@el4cteo/rbx-studio-mcp 0.8.3 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +24 -6
  2. package/dist/bridge/rpc.js +1 -1
  3. package/dist/bridge/rpc.js.map +1 -1
  4. package/dist/index.js +7 -1
  5. package/dist/index.js.map +1 -1
  6. package/dist/lib/apidump.js +75 -0
  7. package/dist/lib/apidump.js.map +1 -1
  8. package/dist/lib/errors.js +5 -4
  9. package/dist/lib/errors.js.map +1 -1
  10. package/dist/lib/format.js +92 -25
  11. package/dist/lib/format.js.map +1 -1
  12. package/dist/lib/notices.js +29 -0
  13. package/dist/lib/notices.js.map +1 -0
  14. package/dist/lib/protocol.js.map +1 -1
  15. package/dist/lib/sync.js +1141 -0
  16. package/dist/lib/sync.js.map +1 -0
  17. package/dist/lib/syncplan.js +338 -0
  18. package/dist/lib/syncplan.js.map +1 -0
  19. package/dist/lib/tool.js +9 -2
  20. package/dist/lib/tool.js.map +1 -1
  21. package/dist/tools/discover.js +6 -1
  22. package/dist/tools/discover.js.map +1 -1
  23. package/dist/tools/exec.js +5 -0
  24. package/dist/tools/exec.js.map +1 -1
  25. package/dist/tools/instances.js +2 -2
  26. package/dist/tools/instances.js.map +1 -1
  27. package/dist/tools/perf.js +28 -3
  28. package/dist/tools/perf.js.map +1 -1
  29. package/dist/tools/screenshot.js +7 -3
  30. package/dist/tools/screenshot.js.map +1 -1
  31. package/dist/tools/scripts.js +110 -28
  32. package/dist/tools/scripts.js.map +1 -1
  33. package/dist/tools/sync.js +169 -0
  34. package/dist/tools/sync.js.map +1 -0
  35. package/package.json +4 -4
  36. package/plugin/src/Commands.luau +72 -5
  37. package/plugin/src/Config.luau +65 -65
  38. package/plugin/src/Console.luau +5 -0
  39. package/plugin/src/Dispatch.luau +131 -90
  40. package/plugin/src/ExecRuntime.luau +190 -168
  41. package/plugin/src/LogBuffer.luau +38 -9
  42. package/plugin/src/Paths.luau +42 -0
  43. package/plugin/src/Phrase.luau +41 -0
  44. package/plugin/src/Prompt.luau +23 -3
  45. package/plugin/src/ScriptEdit.luau +94 -8
  46. package/plugin/src/Serialize.luau +16 -1
  47. package/plugin/src/Transport.luau +5 -2
  48. package/plugin/src/Undo.luau +74 -10
  49. package/plugin/src/handlers/Capture.luau +818 -809
  50. package/plugin/src/handlers/Debug.luau +19 -16
  51. package/plugin/src/handlers/Discover.luau +31 -1
  52. package/plugin/src/handlers/Perf.luau +100 -21
  53. package/plugin/src/handlers/Scripts.luau +274 -90
  54. package/plugin/src/handlers/Sync.luau +968 -0
  55. package/plugin/src/init.server.luau +47 -2
  56. package/scripts/build.mjs +14 -0
  57. package/scripts/sync-fake.mjs +191 -0
  58. package/scripts/test-live-sync-scale.mjs +150 -0
  59. package/scripts/test-live-sync.mjs +232 -0
  60. package/scripts/test-live-tools.mjs +6 -1
  61. package/scripts/test-plugin.mjs +17 -0
  62. package/scripts/test-results.mjs +41 -0
  63. package/scripts/test-sync-more.mjs +228 -0
  64. package/scripts/test-sync.mjs +245 -0
  65. package/dist/tools/spatial.js +0 -135
  66. package/dist/tools/spatial.js.map +0 -1
  67. package/dist/tools/upload.js +0 -294
  68. package/dist/tools/upload.js.map +0 -1
@@ -1,809 +1,818 @@
1
- --!strict
2
- --[[
3
- Screenshots, so an agent can look at the place instead of inferring it.
4
-
5
- Everything else this server exposes reads the data model: names, properties,
6
- numbers. None of that answers "does it look right", which for a 3D medium is
7
- most of the question. A part can sit at the correct position, anchored, the
8
- right size, and still be inside a wall.
9
-
10
- The path is four steps, none of them optional. `CaptureService` hands back a
11
- temporary content id rather than pixels; `AssetService` turns that id into an
12
- EditableImage; the image gives up raw RGBA; and PNG and base64 are written by
13
- hand because the engine has neither. See Png.luau.
14
- ]]
15
-
16
- local AssetService = game:GetService("AssetService")
17
- local CaptureService = game:GetService("CaptureService")
18
- local EncodingService = game:GetService("EncodingService")
19
- local Players = game:GetService("Players")
20
- local RunService = game:GetService("RunService")
21
-
22
- local Dispatch = require(script.Parent.Parent.Dispatch)
23
- local Emulation = require(script.Parent.Parent.Emulation)
24
- local Paths = require(script.Parent.Parent.Paths)
25
- local Png = require(script.Parent.Parent.Png)
26
-
27
- -- Wide enough to read a GUI label, small enough that the encode stays quick and
28
- -- the reply does not dominate the conversation it is part of.
29
- local DEFAULT_WIDTH = 800
30
- local MAX_WIDTH = 1600
31
- local MIN_WIDTH = 160
32
-
33
- -- The callback has never taken close to this. It exists so a capture that never
34
- -- calls back fails with something an agent can act on rather than hanging the
35
- -- session until the request deadline.
36
- local CAPTURE_TIMEOUT = 10
37
-
38
- -- Zstd level. 3 is its default and already gets most of the win on screen
39
- -- content; the higher levels cost Studio's main thread for a few percent.
40
- local COMPRESSION_LEVEL = 3
41
-
42
- local Capture = {}
43
-
44
- --[[
45
- How far two channel values may differ and still count as the same colour.
46
-
47
- Generous, because a surface that looks flat is not mathematically flat: PNG
48
- round-tripping and Studio's own subtle gradients move a byte or two.
49
- ]]
50
- local FLAT_SPREAD = 6
51
-
52
- --[[
53
- Fraction of sampled pixels that must match before the frame counts as empty.
54
-
55
- Not 100%. This is the number two wrong versions of this check were missing:
56
- a viewport showing nothing is never perfectly uniform -- Studio leaves its
57
- settings gear and a loading mark in the corner, and those few hundred pixels
58
- were enough to make an "is every pixel the same" test answer no. 99% leaves
59
- room for that furniture while still being far below anything with a scene in
60
- it: a horizon, a part, a skybox gradient or a GUI all move well past 1%.
61
- ]]
62
- local FLAT_SHARE = 0.99
63
-
64
- --[[
65
- Every Nth pixel is looked at, not every pixel.
66
-
67
- A 1656x674 capture is 1.1M pixels, and walking all of them in Luau to answer
68
- a yes/no question costs more than the screenshot did. Sampling one in 16 is
69
- 11000 samples over a full frame -- far more than enough to notice that
70
- something is drawn, and it cannot miss a scene, because a scene is not 16
71
- pixels.
72
- ]]
73
- local SAMPLE_STRIDE = 16
74
-
75
- --[[
76
- Whether the capture is empty: one colour, give or take the corner furniture.
77
-
78
- Two earlier versions of this got it wrong in the same direction, and both
79
- shipped a picture of nothing described as a picture of something.
80
-
81
- The first asked "is every pixel black". Studio only returns pure black when
82
- it is not rendering at all; when the 3D view is simply not the active tab,
83
- the capture is the script editor's background -- a dark navy around 26-35 --
84
- which sailed past a threshold of 8.
85
-
86
- The second asked "is every pixel the same colour". Closer, and still no: the
87
- empty viewport keeps Studio's settings gear in the top-right corner, so a
88
- handful of pixels differ and the answer came back no again. The picture was
89
- 99% one colour and the test wanted 100%.
90
-
91
- So the question is a proportion, not an absolute. What is being detected is
92
- "nothing was drawn into this", and a frame with anything in it -- a part, a
93
- horizon, a midnight skybox, a GUI -- puts far more than 1% of its pixels
94
- somewhere else.
95
- ]]
96
- local function isFlat(rgb: buffer): boolean
97
- local length = buffer.len(rgb)
98
- if length < 3 then
99
- return true
100
- end
101
-
102
- local red = buffer.readu8(rgb, 0)
103
- local green = buffer.readu8(rgb, 1)
104
- local blue = buffer.readu8(rgb, 2)
105
-
106
- local step = 3 * SAMPLE_STRIDE
107
- local samples = 0
108
- local matches = 0
109
- for offset = 0, length - 3, step do
110
- samples += 1
111
- if
112
- math.abs(buffer.readu8(rgb, offset) - red) <= FLAT_SPREAD
113
- and math.abs(buffer.readu8(rgb, offset + 1) - green) <= FLAT_SPREAD
114
- and math.abs(buffer.readu8(rgb, offset + 2) - blue) <= FLAT_SPREAD
115
- then
116
- matches += 1
117
- end
118
- end
119
-
120
- return samples > 0 and (matches / samples) >= FLAT_SHARE
121
- end
122
-
123
- -- Only a content id comes back over the remote, so this is the time to start a
124
- -- LocalScript and take one shot, not to move any pixels.
125
- local CLIENT_TIMEOUT = 25
126
-
127
- --[[
128
- Screenshots during a playtest, taken by the client because only the client
129
- may take them.
130
-
131
- `CaptureService:CaptureScreenshot` refuses outright in the playtest's server
132
- session -- "can only be called on the client" -- and the edit session, which
133
- could call it, has stopped rendering its own data model and times out. Both
134
- sessions this plugin can reach are therefore the wrong one, and for a long
135
- time that was written down here as "screenshots are impossible mid-playtest".
136
-
137
- They are not. The client is reachable the same way `input` reaches it: parent
138
- a LocalScript into the player's PlayerGui and let it report back over a
139
- RemoteEvent. The client cannot make HTTP requests, but it does not need to --
140
- the server session it replicates to already holds an open bridge.
141
-
142
- The client only takes the shot. It cannot read it: `CaptureService` hands
143
- back a temporary texture id, and `CreateEditableImageAsync` refuses those at
144
- script identity -- "cannot currently create editable image from temporary
145
- texture id" -- which no amount of client-side code gets around.
146
-
147
- The id is not client-local, though. It names a texture in the Studio
148
- process, and the *editor* session's plugin runs there with plugin identity,
149
- where the same call opens it without complaint. So the picture is taken in
150
- one session and read in the other, and the pixels never cross a RemoteEvent
151
- at all -- which also removes the size ceiling that a client-side encode had
152
- to stay under. See `Capture.playtestId` and `Capture.decode`; `screenshot`
153
- in the MCP server drives both halves.
154
- ]]
155
- local CLIENT_SOURCE = [==[
156
- local CaptureService = game:GetService("CaptureService")
157
- local RunService = game:GetService("RunService")
158
-
159
- local relay = script
160
- local report = relay:WaitForChild("Report", 10)
161
- if report == nil then
162
- return
163
- end
164
-
165
- -- One rendered frame before the shot. The RemoteEvent is already a child of
166
- -- this script when it lands, so `WaitForChild` returns at once and the capture
167
- -- would otherwise happen on the frame the relay arrived -- which is the frame
168
- -- least likely to have finished drawing. Roblox's own working repro for
169
- -- CaptureService waits on PreRender first, and it costs a sixtieth of a second.
170
- pcall(function()
171
- RunService.PreRender:Wait()
172
- end)
173
-
174
- local contentId = nil
175
- local okShot, shotErr = pcall(function()
176
- CaptureService:CaptureScreenshot(function(id)
177
- contentId = id
178
- end)
179
- end)
180
- if not okShot then
181
- report:FireServer({ ok = false, reason = tostring(shotErr) })
182
- return
183
- end
184
-
185
- local waited = 0
186
- while contentId == nil and waited < 12 do
187
- task.wait(0.05)
188
- waited += 0.05
189
- end
190
- if contentId == nil then
191
- report:FireServer({ ok = false, reason = "the client did not produce a screenshot within 12s" })
192
- return
193
- end
194
-
195
- report:FireServer({ ok = true, contentId = contentId })
196
- ]==]
197
-
198
- --[[
199
- Takes the shot and waits for the id.
200
-
201
- `CaptureService:CaptureScreenshot` answers through a callback rather than
202
- yielding, so this bridges the two: the handler is already running on its own
203
- task and is free to wait.
204
- ]]
205
- local function takeScreenshot(): string
206
- local contentId: string? = nil
207
- local failed: string? = nil
208
-
209
- local ok, err = pcall(function()
210
- CaptureService:CaptureScreenshot(function(id: string)
211
- contentId = id
212
- end)
213
- end)
214
- if not ok then
215
- failed = tostring(err)
216
- end
217
-
218
- if failed ~= nil then
219
- Dispatch.fail(
220
- "CAPTURE_REFUSED",
221
- string.format("Studio refused to take a screenshot: %s", failed)
222
- )
223
- end
224
-
225
- --[[
226
- A frame at a time to begin with, then a slower poll.
227
-
228
- The callback normally fires within a frame or two, and a flat 0.05s wait
229
- billed that to the agent as up to 50ms of doing nothing -- on a call
230
- whose whole point is that it is taken often. `task.wait()` resumes on the
231
- next heartbeat, so the common case now costs a frame; after a quarter of
232
- a second the shot is clearly not coming back promptly and the poll drops
233
- to the cheap interval for the rest of the timeout.
234
- ]]
235
- local waited = 0
236
- while contentId == nil and waited < CAPTURE_TIMEOUT do
237
- waited += if waited < 0.25 then task.wait() else task.wait(0.05)
238
- end
239
-
240
- if contentId == nil then
241
- Dispatch.fail(
242
- "CAPTURE_TIMEOUT",
243
- string.format("Studio did not return a screenshot within %ds.", CAPTURE_TIMEOUT),
244
- "The viewport may be hidden or Studio may be busy; try again."
245
- )
246
- end
247
-
248
- return contentId :: any
249
- end
250
-
251
- --[[
252
- Half a playtest screenshot: the client takes the shot, this returns its id.
253
-
254
- Deliberately does not try to read it. The editor session does that, because
255
- only plugin identity may -- see the note on CLIENT_SOURCE.
256
- ]]
257
- function Capture.playtestId(_params: { [string]: any }): { [string]: any }
258
- if RunService:IsEdit() or not RunService:IsRunning() then
259
- Dispatch.fail(
260
- "NOT_RUNNING",
261
- "This is not a running playtest session, so there is no client to capture from.",
262
- "Address this at the playtest's studioId from `list_studios`."
263
- )
264
- end
265
-
266
- local players = Players:GetPlayers()
267
- if #players == 0 then
268
- Dispatch.fail(
269
- "NO_PLAYER",
270
- "No player is in this session, so there is no client to capture from.",
271
- "Start a playtest with a character using `playtest op=\"play\"`."
272
- )
273
- end
274
- local player = players[1]
275
- local playerGui = player:FindFirstChildOfClass("PlayerGui")
276
- if playerGui == nil then
277
- Dispatch.fail("NO_PLAYER", string.format("%s has no PlayerGui yet.", player.Name))
278
- end
279
-
280
- local remote = Instance.new("RemoteEvent")
281
- remote.Name = "Report"
282
-
283
- local relay = Instance.new("LocalScript")
284
- relay.Name = "MCPCaptureRelay"
285
- relay.Source = CLIENT_SOURCE
286
- remote.Parent = relay
287
-
288
- local answer: { [string]: any }? = nil
289
- local connection = remote.OnServerEvent:Connect(function(_from, payload)
290
- if typeof(payload) == "table" then
291
- answer = payload :: { [string]: any }
292
- end
293
- end)
294
-
295
- -- Parented last, for the same reason as the input relay: the script runs the
296
- -- instant it lands and immediately waits on the RemoteEvent.
297
- relay.Parent = playerGui
298
-
299
- local deadline = os.clock() + CLIENT_TIMEOUT
300
- while answer == nil and os.clock() < deadline do
301
- task.wait(0.05)
302
- end
303
-
304
- connection:Disconnect()
305
- relay:Destroy()
306
-
307
- if answer == nil then
308
- Dispatch.fail(
309
- "CAPTURE_TIMEOUT",
310
- string.format("The client did not answer within %ds.", CLIENT_TIMEOUT),
311
- "The client may still be loading; try again once the character has spawned."
312
- )
313
- end
314
-
315
- local result = answer :: { [string]: any }
316
- if result.ok ~= true then
317
- Dispatch.fail("CAPTURE_REFUSED", string.format("The client could not capture: %s", tostring(result.reason)))
318
- end
319
-
320
- return { contentId = tostring(result.contentId), device = Emulation.deviceId() }
321
- end
322
-
323
- --[[
324
- Widest image the engine will create for us.
325
-
326
- `AssetService:CreateEditableImage` caps at 1024 on either axis, and the
327
- `width` parameter goes to 1600. Above the cap there is no destination to draw
328
- into, so those calls fall back to the Luau loop rather than failing -- a
329
- slower big screenshot, not a missing one.
330
- ]]
331
- local ENGINE_SCALE_LIMIT = 1024
332
-
333
- --[[
334
- Drops alpha. RGBA in, RGB out, no scaling.
335
-
336
- A screenshot has no alpha worth keeping and carrying it would add a quarter
337
- to every byte that follows, so the channel goes here rather than on the wire.
338
-
339
- Copied a word at a time rather than a byte at a time. Each 32-bit read takes
340
- a whole pixel and each write lands three bytes further on, so the alpha byte
341
- spills one place into the next pixel's slot -- where the next iteration
342
- overwrites it. Only the final pixel has nothing following it, which is what
343
- the one byte of slack on the buffer is for; the result is then trimmed to
344
- the exact length so nothing downstream has to know about it.
345
-
346
- Worth the trick: measured on a 799x359 frame, 16ms a byte at a time against
347
- 6-10ms this way, for output verified identical. That is most of what the
348
- engine-side downscale costs, on a call agents make constantly.
349
- ]]
350
- local function stripAlpha(rgba: buffer, width: number, height: number): buffer
351
- local pixels = width * height
352
- local length = pixels * 3
353
-
354
- local wide = buffer.create(length + 1)
355
- for index = 0, pixels - 1 do
356
- buffer.writeu32(wide, index * 3, buffer.readu32(rgba, index * 4))
357
- end
358
-
359
- local out = buffer.create(length)
360
- buffer.copy(out, 0, wide, 0, length)
361
- return out
362
- end
363
-
364
- --[[
365
- Downscales on the engine's side instead of in Luau.
366
-
367
- This is about the picture, not the clock. `SamplingMode` defaults to bilinear,
368
- so a 2x reduction averages the pixels it merges; the Luau loop it replaces
369
- took every other row and column and dropped the rest, which is how one-pixel
370
- GUI text and thin edges vanished from screenshots that existed to answer
371
- "does this look right".
372
-
373
- It was written expecting to be faster too, and it is not -- that guess is
374
- recorded here because the measurement is the useful part. On a 1370x616 frame
375
- scaled to 799: the old full-resolution read costs about 1ms, not the large
376
- number the 4.4MB of RGBA suggests, and the Luau downscale that follows runs
377
- in 19-22ms all in. `DrawImageTransformed` alone is 16-17ms of engine time.
378
- With `stripAlpha` doing the rest a word at a time, this path lands around
379
- 23ms -- a couple of milliseconds behind, on a call whose round trip is
380
- measured in hundreds. Paying that for legible text is the trade; pretending
381
- it was free would have been the mistake.
382
-
383
- Returns nil when the engine cannot do it -- an older Studio without
384
- `CreateEditableImage`, or a target past ENGINE_SCALE_LIMIT -- and the caller
385
- falls back.
386
- ]]
387
- local function resample(
388
- source: any,
389
- sourceWidth: number,
390
- sourceHeight: number,
391
- targetWidth: number
392
- ): (buffer?, number, number)
393
- local scale = math.min(1, targetWidth / sourceWidth)
394
- local outWidth = math.max(1, math.floor(sourceWidth * scale))
395
- local outHeight = math.max(1, math.floor(sourceHeight * scale))
396
-
397
- if outWidth > ENGINE_SCALE_LIMIT or outHeight > ENGINE_SCALE_LIMIT then
398
- return nil, 0, 0
399
- end
400
-
401
- local ok, rgb = pcall(function()
402
- local canvas = (AssetService :: any):CreateEditableImage({
403
- Size = Vector2.new(outWidth, outHeight),
404
- })
405
- --[[
406
- Position is the centre, not the corner: `DrawImageTransformed` places
407
- the source's PIVOT at the position given, and the pivot defaults to
408
- the middle of the source image. Passing Vector2.zero here puts three
409
- quarters of the frame off-canvas.
410
- ]]
411
- local okDraw = pcall(function()
412
- canvas:DrawImageTransformed(
413
- Vector2.new(outWidth / 2, outHeight / 2),
414
- Vector2.new(scale, scale),
415
- 0,
416
- source,
417
- {
418
- CombineType = Enum.ImageCombineType.Overwrite,
419
- SamplingMode = Enum.ResamplerMode.Default,
420
- }
421
- )
422
- end)
423
- if not okDraw then
424
- canvas:Destroy()
425
- error("draw failed", 0)
426
- end
427
-
428
- local pixels = canvas:ReadPixelsBuffer(Vector2.zero, canvas.Size)
429
- -- Freed here rather than left to the collector: an EditableImage holds
430
- -- its pixels outside Luau's heap, and a screenshot every few seconds
431
- -- would otherwise pile them up for as long as Studio stays open.
432
- canvas:Destroy()
433
- return stripAlpha(pixels, outWidth, outHeight)
434
- end)
435
-
436
- if not ok then
437
- return nil, 0, 0
438
- end
439
- return rgb :: buffer, outWidth, outHeight
440
- end
441
-
442
- --[[
443
- Turns a content id into pixels on the wire.
444
-
445
- Split out from `screenshot` because a playtest shot is taken in one session
446
- and read in another: the id is all that travels, and this is the reading
447
- half, which needs plugin identity and therefore an editor session.
448
- ]]
449
- -- In capture pixels, or in viewport pixels when `viewportWidth` says so.
450
- export type Region = { x: number, y: number, width: number, height: number, viewportWidth: number? }
451
-
452
- --[[
453
- The full-resolution path: the captured pixels themselves, cropped to `region`
454
- when one is given, compressed for the wire and scaled down in Node.
455
-
456
- The engine resample below is bilinear, which only averages the pixels it
457
- merges at exactly 2x -- beyond that it skips source pixels, and a 1661-wide
458
- viewport scaled to 800 lost thin edges and broke small GUI text apart. Node
459
- scales with a box filter instead, where every source pixel counts. Measured
460
- cost of sending the full frame: ~60ms of a ~550ms call, most of which is the
461
- engine's own capture.
462
-
463
- Returns nil when this Studio cannot do it (no EncodingService), so the caller
464
- falls back to the engine resample.
465
- ]]
466
- local function encodeFull(
467
- image: any,
468
- sourceWidth: number,
469
- sourceHeight: number,
470
- width: number,
471
- context: string,
472
- region: Region?
473
- ): { [string]: any }?
474
- local x, y, w, h = 0, 0, sourceWidth, sourceHeight
475
- if region then
476
- -- Viewport and capture pixels differ under display scaling.
477
- local ratio = if region.viewportWidth then sourceWidth / region.viewportWidth else 1
478
- region = {
479
- x = region.x * ratio,
480
- y = region.y * ratio,
481
- width = region.width * ratio,
482
- height = region.height * ratio,
483
- }
484
- x = math.clamp(math.floor(region.x), 0, sourceWidth)
485
- y = math.clamp(math.floor(region.y), 0, sourceHeight)
486
- w = math.min(math.ceil(region.width), sourceWidth - x)
487
- h = math.min(math.ceil(region.height), sourceHeight - y)
488
- if w < 1 or h < 1 then
489
- Dispatch.fail(
490
- "REGION_OFFSCREEN",
491
- string.format(
492
- "The region (%d, %d, %d x %d) is outside the %d x %d viewport.",
493
- region.x,
494
- region.y,
495
- region.width,
496
- region.height,
497
- sourceWidth,
498
- sourceHeight
499
- ),
500
- "Aim the camera at it first with `viewport op=\"focus\"`, then take the screenshot."
501
- )
502
- end
503
- end
504
-
505
- local ok, result = pcall(function()
506
- local rgba = image:ReadPixelsBuffer(Vector2.new(x, y), Vector2.new(w, h))
507
- local rgb = stripAlpha(rgba, w, h)
508
- local compressed = (EncodingService :: any):CompressBuffer(
509
- rgb,
510
- (Enum :: any).CompressionAlgorithm.Zstd,
511
- COMPRESSION_LEVEL
512
- )
513
- return {
514
- rgb = rgb,
515
- packed = buffer.tostring((EncodingService :: any):Base64Encode(compressed)),
516
- }
517
- end)
518
- if not ok then
519
- return nil
520
- end
521
-
522
- return {
523
- encoding = "zstd-rgb",
524
- data = result.packed,
525
- width = w,
526
- height = h,
527
- -- Node scales to this with a box filter; see above.
528
- scaleTo = width,
529
- sourceWidth = sourceWidth,
530
- sourceHeight = sourceHeight,
531
- region = if region then { x = x, y = y, width = w, height = h } else nil,
532
- rawBytes = buffer.len(result.rgb),
533
- bytes = #result.packed,
534
- context = context,
535
- black = isFlat(result.rgb),
536
- device = Emulation.deviceId(),
537
- }
538
- end
539
-
540
- local function encode(contentId: string, width: number, context: string, region: Region?): { [string]: any }
541
- local okImage, image = pcall(function()
542
- return AssetService:CreateEditableImageAsync(Content.fromUri(contentId))
543
- end)
544
- if not okImage then
545
- Dispatch.fail(
546
- "CAPTURE_UNREADABLE",
547
- string.format("The screenshot could not be opened for reading: %s", tostring(image))
548
- )
549
- end
550
-
551
- local size = (image :: any).Size
552
- local sourceWidth = math.floor(size.X)
553
- local sourceHeight = math.floor(size.Y)
554
-
555
- local full = encodeFull(image, sourceWidth, sourceHeight, width, context, region)
556
- if full then
557
- pcall(function()
558
- (image :: any):Destroy()
559
- end)
560
- return full
561
- end
562
- if region then
563
- Dispatch.fail(
564
- "REGION_UNSUPPORTED",
565
- "This Studio build cannot crop a screenshot (EncodingService is missing).",
566
- "Take the full screenshot instead, or update Studio."
567
- )
568
- end
569
-
570
- local rgb, outWidth, outHeight = resample(image, sourceWidth, sourceHeight, width)
571
-
572
- if rgb == nil then
573
- --[[
574
- Both arguments are required. Called with none it reports "expects 2
575
- arguments" rather than defaulting to the whole image, which is worth
576
- stating because every other read on this object takes none.
577
- ]]
578
- local okPixels, pixels = pcall(function()
579
- return (image :: any):ReadPixelsBuffer(Vector2.zero, size)
580
- end)
581
- if not okPixels then
582
- Dispatch.fail(
583
- "CAPTURE_UNREADABLE",
584
- string.format("The screenshot's pixels could not be read: %s", tostring(pixels))
585
- )
586
- end
587
- rgb, outWidth, outHeight = Png.downscaleToRgb(pixels, sourceWidth, sourceHeight, width)
588
- end
589
-
590
- -- On the scaled image, not the source: the question is whether anything was
591
- -- drawn, which survives the downscale, and it is a quarter of the pixels to
592
- -- walk.
593
- local black = isFlat(rgb :: buffer)
594
-
595
- --[[
596
- Raw pixels, compressed by the engine, assembled into a PNG by Node.
597
-
598
- The plugin used to write the whole PNG itself, and could only write a bad
599
- one: Studio has no deflate, so `Png.encode` emitted zlib *stored* blocks
600
- -- the format's "compression not applied" escape hatch -- and every
601
- screenshot travelled and landed at full uncompressed size. It was also
602
- base64-ing by hand, a table lookup per byte in Luau.
603
-
604
- `EncodingService` has both, natively. Zstd is the only algorithm the
605
- engine exposes and PNG cannot use it, so the split is: the plugin
606
- compresses the pixels for the wire, and Node -- which has real zlib --
607
- decompresses and writes a properly deflated PNG. The side with the
608
- compressor does the compressing.
609
-
610
- Guarded rather than assumed: EncodingService is recent, and a Studio
611
- without it should cost a bigger screenshot, not a failed one.
612
- ]]
613
- local okPacked, packed = pcall(function()
614
- local compressed = (EncodingService :: any):CompressBuffer(
615
- rgb :: buffer,
616
- (Enum :: any).CompressionAlgorithm.Zstd,
617
- COMPRESSION_LEVEL
618
- )
619
- return buffer.tostring((EncodingService :: any):Base64Encode(compressed))
620
- end)
621
-
622
- if okPacked then
623
- return {
624
- encoding = "zstd-rgb",
625
- data = packed,
626
- width = outWidth,
627
- height = outHeight,
628
- sourceWidth = sourceWidth,
629
- sourceHeight = sourceHeight,
630
- rawBytes = buffer.len(rgb :: buffer),
631
- bytes = #(packed :: string),
632
- context = context,
633
- black = black,
634
- device = Emulation.deviceId(),
635
- }
636
- end
637
-
638
- local png = Png.encode(rgb :: buffer, outWidth, outHeight)
639
-
640
- return {
641
- encoding = "png",
642
- data = Png.base64(png),
643
- width = outWidth,
644
- height = outHeight,
645
- sourceWidth = sourceWidth,
646
- sourceHeight = sourceHeight,
647
- bytes = buffer.len(png),
648
- context = context,
649
- black = black,
650
- device = Emulation.deviceId(),
651
- }
652
- end
653
-
654
- -- Room left around a zoomed subject, as a share of its size, so its edges are
655
- -- visible rather than cut exactly at the frame.
656
- local REGION_PADDING = 0.08
657
-
658
- --[[
659
- Where on screen a GUI element or a piece of the world is, in viewport pixels.
660
-
661
- GUI positions leave out the top inset, so it is added back; a 3D subject is
662
- the screen box around its bounding box's eight corners. A subject partly
663
- behind the camera has no honest box, so that is refused with the fix.
664
- ]]
665
- local function boundsOf(target: Instance): (Vector2, Vector2)
666
- if target:IsA("GuiObject") then
667
- local inset = game:GetService("GuiService"):GetGuiInset()
668
- return target.AbsolutePosition + inset, target.AbsolutePosition + target.AbsoluteSize + inset
669
- end
670
-
671
- local cframe: CFrame, size: Vector3
672
- if target:IsA("BasePart") then
673
- cframe, size = target.CFrame, target.Size
674
- elseif target:IsA("Model") then
675
- cframe, size = target:GetBoundingBox()
676
- else
677
- local low, high = Vector3.one * math.huge, -Vector3.one * math.huge
678
- local counted = 0
679
- for _, descendant in target:GetDescendants() do
680
- if descendant:IsA("BasePart") then
681
- local half = descendant.Size / 2
682
- for _, corner in { Vector3.new(-1, -1, -1), Vector3.new(1, 1, 1), Vector3.new(-1, 1, -1), Vector3.new(1, -1, 1) } do
683
- local point = descendant.CFrame:PointToWorldSpace(half * corner)
684
- low, high = low:Min(point), high:Max(point)
685
- end
686
- counted += 1
687
- if counted >= 2000 then
688
- break
689
- end
690
- end
691
- end
692
- if counted == 0 then
693
- Dispatch.fail(
694
- "NOT_VISUAL",
695
- string.format("%s has no GUI or parts to zoom to.", target:GetFullName()),
696
- "Zoom to a GuiObject, a BasePart, a Model, or a folder containing parts."
697
- )
698
- end
699
- cframe, size = CFrame.new((low + high) / 2), high - low
700
- end
701
-
702
- local camera = workspace.CurrentCamera
703
- local low, high = Vector2.one * math.huge, -Vector2.one * math.huge
704
- for _, sx in { -1, 1 } do
705
- for _, sy in { -1, 1 } do
706
- for _, sz in { -1, 1 } do
707
- local point, _ = camera:WorldToViewportPoint(cframe:PointToWorldSpace(size / 2 * Vector3.new(sx, sy, sz)))
708
- if point.Z <= 0 then
709
- Dispatch.fail(
710
- "REGION_OFFSCREEN",
711
- string.format("%s is partly behind the camera.", target:GetFullName()),
712
- "Aim the camera at it first with `viewport op=\"focus\"`, then take the screenshot."
713
- )
714
- end
715
- local flat = Vector2.new(point.X, point.Y)
716
- low, high = low:Min(flat), high:Max(flat)
717
- end
718
- end
719
- end
720
- return low, high
721
- end
722
-
723
- --[[
724
- The part of the capture to keep, in capture pixels.
725
-
726
- `path` zooms to an instance; `rect` is "x, y, width, height" in the pixels of
727
- the full-resolution capture, which every screenshot caption states. A path
728
- is resolved before the capture, so a bad one fails without taking it.
729
- ]]
730
- local function regionFor(params: { [string]: any }): Region?
731
- if typeof(params.rect) == "string" and params.rect ~= "" then
732
- local numbers = {}
733
- for token in string.gmatch(params.rect, "-?[%d%.]+") do
734
- table.insert(numbers, tonumber(token))
735
- end
736
- if #numbers ~= 4 or numbers[3] <= 0 or numbers[4] <= 0 then
737
- Dispatch.fail("BAD_PARAMS", 'rect must be "x, y, width, height", e.g. "400, 200, 320, 180".')
738
- end
739
- return { x = numbers[1], y = numbers[2], width = numbers[3], height = numbers[4] }
740
- end
741
-
742
- if typeof(params.path) ~= "string" or params.path == "" then
743
- return nil
744
- end
745
- local target = Paths.resolve(params.path)
746
- local low, high = boundsOf(target)
747
- local extent = high - low
748
- local pad = Vector2.new(math.max(8, extent.X * REGION_PADDING), math.max(8, extent.Y * REGION_PADDING))
749
- low, high = low - pad, high + pad
750
- return {
751
- x = low.X,
752
- y = low.Y,
753
- width = high.X - low.X,
754
- height = high.Y - low.Y,
755
- viewportWidth = workspace.CurrentCamera.ViewportSize.X,
756
- }
757
- end
758
-
759
- function Capture.screenshot(params: { [string]: any }): { [string]: any }
760
- --[[
761
- Refused rather than half-done. A playtest screenshot needs two sessions
762
- and only the MCP server can see both, so failing here with the reason is
763
- better than returning the editor's own idle viewport, which looks like a
764
- valid answer to "what does the player see".
765
- ]]
766
- if RunService:IsRunning() and not RunService:IsEdit() then
767
- Dispatch.fail(
768
- "NEEDS_EDITOR_SESSION",
769
- "A playtest screenshot is taken on the client and read in the editor session, "
770
- .. "so it cannot be served from the playtest server alone.",
771
- "Use the `screenshot` tool, which drives both halves. It needs the editor "
772
- .. "session for the same place to still be connected."
773
- )
774
- end
775
-
776
- local width = math.clamp(tonumber(params.width) or DEFAULT_WIDTH, MIN_WIDTH, MAX_WIDTH)
777
- -- Before the capture, so a path that does not resolve costs no screenshot.
778
- local region = regionFor(params)
779
- return encode(takeScreenshot(), width, if RunService:IsEdit() then "edit" else "playtest", region)
780
- end
781
-
782
- --[[
783
- Reads a content id someone else captured.
784
-
785
- Only the editor session can do this, and only because the temporary texture
786
- is owned by the Studio process rather than by the session that made it.
787
- ]]
788
- function Capture.decode(params: { [string]: any }): { [string]: any }
789
- local contentId = params.contentId
790
- if typeof(contentId) ~= "string" or contentId == "" then
791
- Dispatch.fail("BAD_PARAMS", "decode needs a `contentId`.")
792
- end
793
-
794
- local width = math.clamp(tonumber(params.width) or DEFAULT_WIDTH, MIN_WIDTH, MAX_WIDTH)
795
- local context = if typeof(params.context) == "string" then params.context else "playtest client"
796
- -- Only `rect` here: a path would have to be resolved in the client's view.
797
- local region = if typeof(params.rect) == "string" then regionFor({ rect = params.rect }) else nil
798
- return encode(contentId :: string, width, context, region)
799
- end
800
-
801
- function Capture.register()
802
- Dispatch.registerAll("capture", {
803
- screenshot = Capture.screenshot,
804
- playtestId = Capture.playtestId,
805
- decode = Capture.decode,
806
- })
807
- end
808
-
809
- return Capture
1
+ --!strict
2
+ --[[
3
+ Screenshots, so an agent can look at the place instead of inferring it.
4
+
5
+ Everything else this server exposes reads the data model: names, properties,
6
+ numbers. None of that answers "does it look right", which for a 3D medium is
7
+ most of the question. A part can sit at the correct position, anchored, the
8
+ right size, and still be inside a wall.
9
+
10
+ The path is four steps, none of them optional. `CaptureService` hands back a
11
+ temporary content id rather than pixels; `AssetService` turns that id into an
12
+ EditableImage; the image gives up raw RGBA; and PNG and base64 are written by
13
+ hand because the engine has neither. See Png.luau.
14
+ ]]
15
+
16
+ local AssetService = game:GetService("AssetService")
17
+ local CaptureService = game:GetService("CaptureService")
18
+ local EncodingService = game:GetService("EncodingService")
19
+ local Players = game:GetService("Players")
20
+ local RunService = game:GetService("RunService")
21
+
22
+ local Dispatch = require(script.Parent.Parent.Dispatch)
23
+ local Emulation = require(script.Parent.Parent.Emulation)
24
+ local Paths = require(script.Parent.Parent.Paths)
25
+ local Png = require(script.Parent.Parent.Png)
26
+
27
+ -- Wide enough to read a GUI label, small enough that the encode stays quick and
28
+ -- the reply does not dominate the conversation it is part of.
29
+ local DEFAULT_WIDTH = 800
30
+ local MAX_WIDTH = 1600
31
+ local MIN_WIDTH = 160
32
+
33
+ -- The callback has never taken close to this. It exists so a capture that never
34
+ -- calls back fails with something an agent can act on rather than hanging the
35
+ -- session until the request deadline.
36
+ local CAPTURE_TIMEOUT = 10
37
+
38
+ -- Zstd level. 3 is its default and already gets most of the win on screen
39
+ -- content; the higher levels cost Studio's main thread for a few percent.
40
+ local COMPRESSION_LEVEL = 3
41
+
42
+ local Capture = {}
43
+
44
+ --[[
45
+ How far two channel values may differ and still count as the same colour.
46
+
47
+ Generous, because a surface that looks flat is not mathematically flat: PNG
48
+ round-tripping and Studio's own subtle gradients move a byte or two.
49
+ ]]
50
+ local FLAT_SPREAD = 6
51
+
52
+ --[[
53
+ Fraction of sampled pixels that must match before the frame counts as empty.
54
+
55
+ Not 100%. This is the number two wrong versions of this check were missing:
56
+ a viewport showing nothing is never perfectly uniform -- Studio leaves its
57
+ settings gear and a loading mark in the corner, and those few hundred pixels
58
+ were enough to make an "is every pixel the same" test answer no. 99% leaves
59
+ room for that furniture while still being far below anything with a scene in
60
+ it: a horizon, a part, a skybox gradient or a GUI all move well past 1%.
61
+ ]]
62
+ local FLAT_SHARE = 0.99
63
+
64
+ --[[
65
+ Every Nth pixel is looked at, not every pixel.
66
+
67
+ A 1656x674 capture is 1.1M pixels, and walking all of them in Luau to answer
68
+ a yes/no question costs more than the screenshot did. Sampling one in 16 is
69
+ 11000 samples over a full frame -- far more than enough to notice that
70
+ something is drawn, and it cannot miss a scene, because a scene is not 16
71
+ pixels.
72
+ ]]
73
+ local SAMPLE_STRIDE = 16
74
+
75
+ --[[
76
+ Whether the capture is empty: one colour, give or take the corner furniture.
77
+
78
+ Two earlier versions of this got it wrong in the same direction, and both
79
+ shipped a picture of nothing described as a picture of something.
80
+
81
+ The first asked "is every pixel black". Studio only returns pure black when
82
+ it is not rendering at all; when the 3D view is simply not the active tab,
83
+ the capture is the script editor's background -- a dark navy around 26-35 --
84
+ which sailed past a threshold of 8.
85
+
86
+ The second asked "is every pixel the same colour". Closer, and still no: the
87
+ empty viewport keeps Studio's settings gear in the top-right corner, so a
88
+ handful of pixels differ and the answer came back no again. The picture was
89
+ 99% one colour and the test wanted 100%.
90
+
91
+ So the question is a proportion, not an absolute. What is being detected is
92
+ "nothing was drawn into this", and a frame with anything in it -- a part, a
93
+ horizon, a midnight skybox, a GUI -- puts far more than 1% of its pixels
94
+ somewhere else.
95
+ ]]
96
+ local function isFlat(rgb: buffer): boolean
97
+ local length = buffer.len(rgb)
98
+ if length < 3 then
99
+ return true
100
+ end
101
+
102
+ local red = buffer.readu8(rgb, 0)
103
+ local green = buffer.readu8(rgb, 1)
104
+ local blue = buffer.readu8(rgb, 2)
105
+
106
+ local step = 3 * SAMPLE_STRIDE
107
+ local samples = 0
108
+ local matches = 0
109
+ for offset = 0, length - 3, step do
110
+ samples += 1
111
+ if
112
+ math.abs(buffer.readu8(rgb, offset) - red) <= FLAT_SPREAD
113
+ and math.abs(buffer.readu8(rgb, offset + 1) - green) <= FLAT_SPREAD
114
+ and math.abs(buffer.readu8(rgb, offset + 2) - blue) <= FLAT_SPREAD
115
+ then
116
+ matches += 1
117
+ end
118
+ end
119
+
120
+ return samples > 0 and (matches / samples) >= FLAT_SHARE
121
+ end
122
+
123
+ -- Only a content id comes back over the remote, so this is the time to start a
124
+ -- LocalScript and take one shot, not to move any pixels.
125
+ local CLIENT_TIMEOUT = 25
126
+
127
+ --[[
128
+ Screenshots during a playtest, taken by the client because only the client
129
+ may take them.
130
+
131
+ `CaptureService:CaptureScreenshot` refuses outright in the playtest's server
132
+ session -- "can only be called on the client" -- and the edit session, which
133
+ could call it, has stopped rendering its own data model and times out. Both
134
+ sessions this plugin can reach are therefore the wrong one, and for a long
135
+ time that was written down here as "screenshots are impossible mid-playtest".
136
+
137
+ They are not. The client is reachable the same way `input` reaches it: parent
138
+ a LocalScript into the player's PlayerGui and let it report back over a
139
+ RemoteEvent. The client cannot make HTTP requests, but it does not need to --
140
+ the server session it replicates to already holds an open bridge.
141
+
142
+ The client only takes the shot. It cannot read it: `CaptureService` hands
143
+ back a temporary texture id, and `CreateEditableImageAsync` refuses those at
144
+ script identity -- "cannot currently create editable image from temporary
145
+ texture id" -- which no amount of client-side code gets around.
146
+
147
+ The id is not client-local, though. It names a texture in the Studio
148
+ process, and the *editor* session's plugin runs there with plugin identity,
149
+ where the same call opens it without complaint. So the picture is taken in
150
+ one session and read in the other, and the pixels never cross a RemoteEvent
151
+ at all -- which also removes the size ceiling that a client-side encode had
152
+ to stay under. See `Capture.playtestId` and `Capture.decode`; `screenshot`
153
+ in the MCP server drives both halves.
154
+ ]]
155
+ local CLIENT_SOURCE = [==[
156
+ local CaptureService = game:GetService("CaptureService")
157
+ local RunService = game:GetService("RunService")
158
+
159
+ local relay = script
160
+ local report = relay:WaitForChild("Report", 10)
161
+ if report == nil then
162
+ return
163
+ end
164
+
165
+ -- One rendered frame before the shot. The RemoteEvent is already a child of
166
+ -- this script when it lands, so `WaitForChild` returns at once and the capture
167
+ -- would otherwise happen on the frame the relay arrived -- which is the frame
168
+ -- least likely to have finished drawing. Roblox's own working repro for
169
+ -- CaptureService waits on PreRender first, and it costs a sixtieth of a second.
170
+ pcall(function()
171
+ RunService.PreRender:Wait()
172
+ end)
173
+
174
+ local contentId = nil
175
+ local okShot, shotErr = pcall(function()
176
+ CaptureService:CaptureScreenshot(function(id)
177
+ contentId = id
178
+ end)
179
+ end)
180
+ if not okShot then
181
+ report:FireServer({ ok = false, reason = tostring(shotErr) })
182
+ return
183
+ end
184
+
185
+ local waited = 0
186
+ while contentId == nil and waited < 12 do
187
+ task.wait(0.05)
188
+ waited += 0.05
189
+ end
190
+ if contentId == nil then
191
+ report:FireServer({ ok = false, reason = "the client did not produce a screenshot within 12s" })
192
+ return
193
+ end
194
+
195
+ report:FireServer({ ok = true, contentId = contentId })
196
+ ]==]
197
+
198
+ --[[
199
+ Takes the shot and waits for the id.
200
+
201
+ `CaptureService:CaptureScreenshot` answers through a callback rather than
202
+ yielding, so this bridges the two: the handler is already running on its own
203
+ task and is free to wait.
204
+ ]]
205
+ local function takeScreenshot(): string
206
+ local contentId: string? = nil
207
+ local failed: string? = nil
208
+
209
+ local ok, err = pcall(function()
210
+ CaptureService:CaptureScreenshot(function(id: string)
211
+ contentId = id
212
+ end)
213
+ end)
214
+ if not ok then
215
+ failed = tostring(err)
216
+ end
217
+
218
+ if failed ~= nil then
219
+ Dispatch.fail(
220
+ "CAPTURE_REFUSED",
221
+ string.format("Studio refused to take a screenshot: %s", failed)
222
+ )
223
+ end
224
+
225
+ --[[
226
+ A frame at a time to begin with, then a slower poll.
227
+
228
+ The callback normally fires within a frame or two, and a flat 0.05s wait
229
+ billed that to the agent as up to 50ms of doing nothing -- on a call
230
+ whose whole point is that it is taken often. `task.wait()` resumes on the
231
+ next heartbeat, so the common case now costs a frame; after a quarter of
232
+ a second the shot is clearly not coming back promptly and the poll drops
233
+ to the cheap interval for the rest of the timeout.
234
+ ]]
235
+ local waited = 0
236
+ while contentId == nil and waited < CAPTURE_TIMEOUT do
237
+ waited += if waited < 0.25 then task.wait() else task.wait(0.05)
238
+ end
239
+
240
+ if contentId == nil then
241
+ Dispatch.fail(
242
+ "CAPTURE_TIMEOUT",
243
+ string.format("Studio did not return a screenshot within %ds.", CAPTURE_TIMEOUT),
244
+ "The viewport may be hidden or Studio may be busy; try again."
245
+ )
246
+ end
247
+
248
+ return contentId :: any
249
+ end
250
+
251
+ --[[
252
+ Half a playtest screenshot: the client takes the shot, this returns its id.
253
+
254
+ Deliberately does not try to read it. The editor session does that, because
255
+ only plugin identity may -- see the note on CLIENT_SOURCE.
256
+ ]]
257
+ function Capture.playtestId(params: { [string]: any }): { [string]: any }
258
+ if RunService:IsEdit() or not RunService:IsRunning() then
259
+ Dispatch.fail(
260
+ "NOT_RUNNING",
261
+ "This is not a running playtest session, so there is no client to capture from.",
262
+ "Address this at the playtest's studioId from `list_studios`."
263
+ )
264
+ end
265
+
266
+ local players = Players:GetPlayers()
267
+ if #players == 0 then
268
+ Dispatch.fail(
269
+ "NO_PLAYER",
270
+ "No player is in this session, so there is no client to capture from.",
271
+ "Start a playtest with a character using `playtest op=\"play\"`."
272
+ )
273
+ end
274
+ local player: Player? = nil
275
+ if params.player then
276
+ for _, candidate in players do if candidate.Name == params.player then player = candidate end end
277
+ if not player then Dispatch.fail("NO_PLAYER", "No player named " .. tostring(params.player)) end
278
+ elseif #players == 1 then
279
+ player = players[1]
280
+ else
281
+ Dispatch.fail("AMBIGUOUS_PLAYER", "Select a screenshot player by name.")
282
+ end
283
+ player = player :: Player
284
+ local playerGui = player:FindFirstChildOfClass("PlayerGui")
285
+ if playerGui == nil then
286
+ Dispatch.fail("NO_PLAYER", string.format("%s has no PlayerGui yet.", player.Name))
287
+ end
288
+
289
+ local remote = Instance.new("RemoteEvent")
290
+ remote.Name = "Report"
291
+
292
+ local relay = Instance.new("LocalScript")
293
+ relay.Name = "MCPCaptureRelay"
294
+ relay.Source = CLIENT_SOURCE
295
+ remote.Parent = relay
296
+
297
+ local answer: { [string]: any }? = nil
298
+ local connection = remote.OnServerEvent:Connect(function(_from, payload)
299
+ if typeof(payload) == "table" then
300
+ answer = payload :: { [string]: any }
301
+ end
302
+ end)
303
+
304
+ -- Parented last, for the same reason as the input relay: the script runs the
305
+ -- instant it lands and immediately waits on the RemoteEvent.
306
+ relay.Parent = playerGui
307
+
308
+ local deadline = os.clock() + CLIENT_TIMEOUT
309
+ while answer == nil and os.clock() < deadline do
310
+ task.wait(0.05)
311
+ end
312
+
313
+ connection:Disconnect()
314
+ relay:Destroy()
315
+
316
+ if answer == nil then
317
+ Dispatch.fail(
318
+ "CAPTURE_TIMEOUT",
319
+ string.format("The client did not answer within %ds.", CLIENT_TIMEOUT),
320
+ "The client may still be loading; try again once the character has spawned."
321
+ )
322
+ end
323
+
324
+ local result = answer :: { [string]: any }
325
+ if result.ok ~= true then
326
+ Dispatch.fail("CAPTURE_REFUSED", string.format("The client could not capture: %s", tostring(result.reason)))
327
+ end
328
+
329
+ return { contentId = tostring(result.contentId), device = Emulation.deviceId() }
330
+ end
331
+
332
+ --[[
333
+ Widest image the engine will create for us.
334
+
335
+ `AssetService:CreateEditableImage` caps at 1024 on either axis, and the
336
+ `width` parameter goes to 1600. Above the cap there is no destination to draw
337
+ into, so those calls fall back to the Luau loop rather than failing -- a
338
+ slower big screenshot, not a missing one.
339
+ ]]
340
+ local ENGINE_SCALE_LIMIT = 1024
341
+
342
+ --[[
343
+ Drops alpha. RGBA in, RGB out, no scaling.
344
+
345
+ A screenshot has no alpha worth keeping and carrying it would add a quarter
346
+ to every byte that follows, so the channel goes here rather than on the wire.
347
+
348
+ Copied a word at a time rather than a byte at a time. Each 32-bit read takes
349
+ a whole pixel and each write lands three bytes further on, so the alpha byte
350
+ spills one place into the next pixel's slot -- where the next iteration
351
+ overwrites it. Only the final pixel has nothing following it, which is what
352
+ the one byte of slack on the buffer is for; the result is then trimmed to
353
+ the exact length so nothing downstream has to know about it.
354
+
355
+ Worth the trick: measured on a 799x359 frame, 16ms a byte at a time against
356
+ 6-10ms this way, for output verified identical. That is most of what the
357
+ engine-side downscale costs, on a call agents make constantly.
358
+ ]]
359
+ local function stripAlpha(rgba: buffer, width: number, height: number): buffer
360
+ local pixels = width * height
361
+ local length = pixels * 3
362
+
363
+ local wide = buffer.create(length + 1)
364
+ for index = 0, pixels - 1 do
365
+ buffer.writeu32(wide, index * 3, buffer.readu32(rgba, index * 4))
366
+ end
367
+
368
+ local out = buffer.create(length)
369
+ buffer.copy(out, 0, wide, 0, length)
370
+ return out
371
+ end
372
+
373
+ --[[
374
+ Downscales on the engine's side instead of in Luau.
375
+
376
+ This is about the picture, not the clock. `SamplingMode` defaults to bilinear,
377
+ so a 2x reduction averages the pixels it merges; the Luau loop it replaces
378
+ took every other row and column and dropped the rest, which is how one-pixel
379
+ GUI text and thin edges vanished from screenshots that existed to answer
380
+ "does this look right".
381
+
382
+ It was written expecting to be faster too, and it is not -- that guess is
383
+ recorded here because the measurement is the useful part. On a 1370x616 frame
384
+ scaled to 799: the old full-resolution read costs about 1ms, not the large
385
+ number the 4.4MB of RGBA suggests, and the Luau downscale that follows runs
386
+ in 19-22ms all in. `DrawImageTransformed` alone is 16-17ms of engine time.
387
+ With `stripAlpha` doing the rest a word at a time, this path lands around
388
+ 23ms -- a couple of milliseconds behind, on a call whose round trip is
389
+ measured in hundreds. Paying that for legible text is the trade; pretending
390
+ it was free would have been the mistake.
391
+
392
+ Returns nil when the engine cannot do it -- an older Studio without
393
+ `CreateEditableImage`, or a target past ENGINE_SCALE_LIMIT -- and the caller
394
+ falls back.
395
+ ]]
396
+ local function resample(
397
+ source: any,
398
+ sourceWidth: number,
399
+ sourceHeight: number,
400
+ targetWidth: number
401
+ ): (buffer?, number, number)
402
+ local scale = math.min(1, targetWidth / sourceWidth)
403
+ local outWidth = math.max(1, math.floor(sourceWidth * scale))
404
+ local outHeight = math.max(1, math.floor(sourceHeight * scale))
405
+
406
+ if outWidth > ENGINE_SCALE_LIMIT or outHeight > ENGINE_SCALE_LIMIT then
407
+ return nil, 0, 0
408
+ end
409
+
410
+ local ok, rgb = pcall(function()
411
+ local canvas = (AssetService :: any):CreateEditableImage({
412
+ Size = Vector2.new(outWidth, outHeight),
413
+ })
414
+ --[[
415
+ Position is the centre, not the corner: `DrawImageTransformed` places
416
+ the source's PIVOT at the position given, and the pivot defaults to
417
+ the middle of the source image. Passing Vector2.zero here puts three
418
+ quarters of the frame off-canvas.
419
+ ]]
420
+ local okDraw = pcall(function()
421
+ canvas:DrawImageTransformed(
422
+ Vector2.new(outWidth / 2, outHeight / 2),
423
+ Vector2.new(scale, scale),
424
+ 0,
425
+ source,
426
+ {
427
+ CombineType = Enum.ImageCombineType.Overwrite,
428
+ SamplingMode = Enum.ResamplerMode.Default,
429
+ }
430
+ )
431
+ end)
432
+ if not okDraw then
433
+ canvas:Destroy()
434
+ error("draw failed", 0)
435
+ end
436
+
437
+ local pixels = canvas:ReadPixelsBuffer(Vector2.zero, canvas.Size)
438
+ -- Freed here rather than left to the collector: an EditableImage holds
439
+ -- its pixels outside Luau's heap, and a screenshot every few seconds
440
+ -- would otherwise pile them up for as long as Studio stays open.
441
+ canvas:Destroy()
442
+ return stripAlpha(pixels, outWidth, outHeight)
443
+ end)
444
+
445
+ if not ok then
446
+ return nil, 0, 0
447
+ end
448
+ return rgb :: buffer, outWidth, outHeight
449
+ end
450
+
451
+ --[[
452
+ Turns a content id into pixels on the wire.
453
+
454
+ Split out from `screenshot` because a playtest shot is taken in one session
455
+ and read in another: the id is all that travels, and this is the reading
456
+ half, which needs plugin identity and therefore an editor session.
457
+ ]]
458
+ -- In capture pixels, or in viewport pixels when `viewportWidth` says so.
459
+ export type Region = { x: number, y: number, width: number, height: number, viewportWidth: number? }
460
+
461
+ --[[
462
+ The full-resolution path: the captured pixels themselves, cropped to `region`
463
+ when one is given, compressed for the wire and scaled down in Node.
464
+
465
+ The engine resample below is bilinear, which only averages the pixels it
466
+ merges at exactly 2x -- beyond that it skips source pixels, and a 1661-wide
467
+ viewport scaled to 800 lost thin edges and broke small GUI text apart. Node
468
+ scales with a box filter instead, where every source pixel counts. Measured
469
+ cost of sending the full frame: ~60ms of a ~550ms call, most of which is the
470
+ engine's own capture.
471
+
472
+ Returns nil when this Studio cannot do it (no EncodingService), so the caller
473
+ falls back to the engine resample.
474
+ ]]
475
+ local function encodeFull(
476
+ image: any,
477
+ sourceWidth: number,
478
+ sourceHeight: number,
479
+ width: number,
480
+ context: string,
481
+ region: Region?
482
+ ): { [string]: any }?
483
+ local x, y, w, h = 0, 0, sourceWidth, sourceHeight
484
+ if region then
485
+ -- Viewport and capture pixels differ under display scaling.
486
+ local ratio = if region.viewportWidth then sourceWidth / region.viewportWidth else 1
487
+ region = {
488
+ x = region.x * ratio,
489
+ y = region.y * ratio,
490
+ width = region.width * ratio,
491
+ height = region.height * ratio,
492
+ }
493
+ x = math.clamp(math.floor(region.x), 0, sourceWidth)
494
+ y = math.clamp(math.floor(region.y), 0, sourceHeight)
495
+ w = math.min(math.ceil(region.width), sourceWidth - x)
496
+ h = math.min(math.ceil(region.height), sourceHeight - y)
497
+ if w < 1 or h < 1 then
498
+ Dispatch.fail(
499
+ "REGION_OFFSCREEN",
500
+ string.format(
501
+ "The region (%d, %d, %d x %d) is outside the %d x %d viewport.",
502
+ region.x,
503
+ region.y,
504
+ region.width,
505
+ region.height,
506
+ sourceWidth,
507
+ sourceHeight
508
+ ),
509
+ "Aim the camera at it first with `viewport op=\"focus\"`, then take the screenshot."
510
+ )
511
+ end
512
+ end
513
+
514
+ local ok, result = pcall(function()
515
+ local rgba = image:ReadPixelsBuffer(Vector2.new(x, y), Vector2.new(w, h))
516
+ local rgb = stripAlpha(rgba, w, h)
517
+ local compressed = (EncodingService :: any):CompressBuffer(
518
+ rgb,
519
+ (Enum :: any).CompressionAlgorithm.Zstd,
520
+ COMPRESSION_LEVEL
521
+ )
522
+ return {
523
+ rgb = rgb,
524
+ packed = buffer.tostring((EncodingService :: any):Base64Encode(compressed)),
525
+ }
526
+ end)
527
+ if not ok then
528
+ return nil
529
+ end
530
+
531
+ return {
532
+ encoding = "zstd-rgb",
533
+ data = result.packed,
534
+ width = w,
535
+ height = h,
536
+ -- Node scales to this with a box filter; see above.
537
+ scaleTo = width,
538
+ sourceWidth = sourceWidth,
539
+ sourceHeight = sourceHeight,
540
+ region = if region then { x = x, y = y, width = w, height = h } else nil,
541
+ rawBytes = buffer.len(result.rgb),
542
+ bytes = #result.packed,
543
+ context = context,
544
+ black = isFlat(result.rgb),
545
+ device = Emulation.deviceId(),
546
+ }
547
+ end
548
+
549
+ local function encode(contentId: string, width: number, context: string, region: Region?): { [string]: any }
550
+ local okImage, image = pcall(function()
551
+ return AssetService:CreateEditableImageAsync(Content.fromUri(contentId))
552
+ end)
553
+ if not okImage then
554
+ Dispatch.fail(
555
+ "CAPTURE_UNREADABLE",
556
+ string.format("The screenshot could not be opened for reading: %s", tostring(image))
557
+ )
558
+ end
559
+
560
+ local size = (image :: any).Size
561
+ local sourceWidth = math.floor(size.X)
562
+ local sourceHeight = math.floor(size.Y)
563
+
564
+ local full = encodeFull(image, sourceWidth, sourceHeight, width, context, region)
565
+ if full then
566
+ pcall(function()
567
+ (image :: any):Destroy()
568
+ end)
569
+ return full
570
+ end
571
+ if region then
572
+ Dispatch.fail(
573
+ "REGION_UNSUPPORTED",
574
+ "This Studio build cannot crop a screenshot (EncodingService is missing).",
575
+ "Take the full screenshot instead, or update Studio."
576
+ )
577
+ end
578
+
579
+ local rgb, outWidth, outHeight = resample(image, sourceWidth, sourceHeight, width)
580
+
581
+ if rgb == nil then
582
+ --[[
583
+ Both arguments are required. Called with none it reports "expects 2
584
+ arguments" rather than defaulting to the whole image, which is worth
585
+ stating because every other read on this object takes none.
586
+ ]]
587
+ local okPixels, pixels = pcall(function()
588
+ return (image :: any):ReadPixelsBuffer(Vector2.zero, size)
589
+ end)
590
+ if not okPixels then
591
+ Dispatch.fail(
592
+ "CAPTURE_UNREADABLE",
593
+ string.format("The screenshot's pixels could not be read: %s", tostring(pixels))
594
+ )
595
+ end
596
+ rgb, outWidth, outHeight = Png.downscaleToRgb(pixels, sourceWidth, sourceHeight, width)
597
+ end
598
+
599
+ -- On the scaled image, not the source: the question is whether anything was
600
+ -- drawn, which survives the downscale, and it is a quarter of the pixels to
601
+ -- walk.
602
+ local black = isFlat(rgb :: buffer)
603
+
604
+ --[[
605
+ Raw pixels, compressed by the engine, assembled into a PNG by Node.
606
+
607
+ The plugin used to write the whole PNG itself, and could only write a bad
608
+ one: Studio has no deflate, so `Png.encode` emitted zlib *stored* blocks
609
+ -- the format's "compression not applied" escape hatch -- and every
610
+ screenshot travelled and landed at full uncompressed size. It was also
611
+ base64-ing by hand, a table lookup per byte in Luau.
612
+
613
+ `EncodingService` has both, natively. Zstd is the only algorithm the
614
+ engine exposes and PNG cannot use it, so the split is: the plugin
615
+ compresses the pixels for the wire, and Node -- which has real zlib --
616
+ decompresses and writes a properly deflated PNG. The side with the
617
+ compressor does the compressing.
618
+
619
+ Guarded rather than assumed: EncodingService is recent, and a Studio
620
+ without it should cost a bigger screenshot, not a failed one.
621
+ ]]
622
+ local okPacked, packed = pcall(function()
623
+ local compressed = (EncodingService :: any):CompressBuffer(
624
+ rgb :: buffer,
625
+ (Enum :: any).CompressionAlgorithm.Zstd,
626
+ COMPRESSION_LEVEL
627
+ )
628
+ return buffer.tostring((EncodingService :: any):Base64Encode(compressed))
629
+ end)
630
+
631
+ if okPacked then
632
+ return {
633
+ encoding = "zstd-rgb",
634
+ data = packed,
635
+ width = outWidth,
636
+ height = outHeight,
637
+ sourceWidth = sourceWidth,
638
+ sourceHeight = sourceHeight,
639
+ rawBytes = buffer.len(rgb :: buffer),
640
+ bytes = #(packed :: string),
641
+ context = context,
642
+ black = black,
643
+ device = Emulation.deviceId(),
644
+ }
645
+ end
646
+
647
+ local png = Png.encode(rgb :: buffer, outWidth, outHeight)
648
+
649
+ return {
650
+ encoding = "png",
651
+ data = Png.base64(png),
652
+ width = outWidth,
653
+ height = outHeight,
654
+ sourceWidth = sourceWidth,
655
+ sourceHeight = sourceHeight,
656
+ bytes = buffer.len(png),
657
+ context = context,
658
+ black = black,
659
+ device = Emulation.deviceId(),
660
+ }
661
+ end
662
+
663
+ -- Room left around a zoomed subject, as a share of its size, so its edges are
664
+ -- visible rather than cut exactly at the frame.
665
+ local REGION_PADDING = 0.08
666
+
667
+ --[[
668
+ Where on screen a GUI element or a piece of the world is, in viewport pixels.
669
+
670
+ GUI positions leave out the top inset, so it is added back; a 3D subject is
671
+ the screen box around its bounding box's eight corners. A subject partly
672
+ behind the camera has no honest box, so that is refused with the fix.
673
+ ]]
674
+ local function boundsOf(target: Instance): (Vector2, Vector2)
675
+ if target:IsA("GuiObject") then
676
+ local inset = game:GetService("GuiService"):GetGuiInset()
677
+ return target.AbsolutePosition + inset, target.AbsolutePosition + target.AbsoluteSize + inset
678
+ end
679
+
680
+ local cframe: CFrame, size: Vector3
681
+ if target:IsA("BasePart") then
682
+ cframe, size = target.CFrame, target.Size
683
+ elseif target:IsA("Model") then
684
+ cframe, size = target:GetBoundingBox()
685
+ else
686
+ local low, high = Vector3.one * math.huge, -Vector3.one * math.huge
687
+ local counted = 0
688
+ for _, descendant in target:GetDescendants() do
689
+ if descendant:IsA("BasePart") then
690
+ local half = descendant.Size / 2
691
+ for _, corner in { Vector3.new(-1, -1, -1), Vector3.new(1, 1, 1), Vector3.new(-1, 1, -1), Vector3.new(1, -1, 1) } do
692
+ local point = descendant.CFrame:PointToWorldSpace(half * corner)
693
+ low, high = low:Min(point), high:Max(point)
694
+ end
695
+ counted += 1
696
+ if counted >= 2000 then
697
+ break
698
+ end
699
+ end
700
+ end
701
+ if counted == 0 then
702
+ Dispatch.fail(
703
+ "NOT_VISUAL",
704
+ string.format("%s has no GUI or parts to zoom to.", target:GetFullName()),
705
+ "Zoom to a GuiObject, a BasePart, a Model, or a folder containing parts."
706
+ )
707
+ end
708
+ cframe, size = CFrame.new((low + high) / 2), high - low
709
+ end
710
+
711
+ local camera = workspace.CurrentCamera
712
+ local low, high = Vector2.one * math.huge, -Vector2.one * math.huge
713
+ for _, sx in { -1, 1 } do
714
+ for _, sy in { -1, 1 } do
715
+ for _, sz in { -1, 1 } do
716
+ local point, _ = camera:WorldToViewportPoint(cframe:PointToWorldSpace(size / 2 * Vector3.new(sx, sy, sz)))
717
+ if point.Z <= 0 then
718
+ Dispatch.fail(
719
+ "REGION_OFFSCREEN",
720
+ string.format("%s is partly behind the camera.", target:GetFullName()),
721
+ "Aim the camera at it first with `viewport op=\"focus\"`, then take the screenshot."
722
+ )
723
+ end
724
+ local flat = Vector2.new(point.X, point.Y)
725
+ low, high = low:Min(flat), high:Max(flat)
726
+ end
727
+ end
728
+ end
729
+ return low, high
730
+ end
731
+
732
+ --[[
733
+ The part of the capture to keep, in capture pixels.
734
+
735
+ `path` zooms to an instance; `rect` is "x, y, width, height" in the pixels of
736
+ the full-resolution capture, which every screenshot caption states. A path
737
+ is resolved before the capture, so a bad one fails without taking it.
738
+ ]]
739
+ local function regionFor(params: { [string]: any }): Region?
740
+ if typeof(params.rect) == "string" and params.rect ~= "" then
741
+ local numbers = {}
742
+ for token in string.gmatch(params.rect, "-?[%d%.]+") do
743
+ table.insert(numbers, tonumber(token))
744
+ end
745
+ if #numbers ~= 4 or numbers[3] <= 0 or numbers[4] <= 0 then
746
+ Dispatch.fail("BAD_PARAMS", 'rect must be "x, y, width, height", e.g. "400, 200, 320, 180".')
747
+ end
748
+ return { x = numbers[1], y = numbers[2], width = numbers[3], height = numbers[4] }
749
+ end
750
+
751
+ if typeof(params.path) ~= "string" or params.path == "" then
752
+ return nil
753
+ end
754
+ local target = Paths.resolve(params.path)
755
+ local low, high = boundsOf(target)
756
+ local extent = high - low
757
+ local pad = Vector2.new(math.max(8, extent.X * REGION_PADDING), math.max(8, extent.Y * REGION_PADDING))
758
+ low, high = low - pad, high + pad
759
+ return {
760
+ x = low.X,
761
+ y = low.Y,
762
+ width = high.X - low.X,
763
+ height = high.Y - low.Y,
764
+ viewportWidth = workspace.CurrentCamera.ViewportSize.X,
765
+ }
766
+ end
767
+
768
+ function Capture.screenshot(params: { [string]: any }): { [string]: any }
769
+ --[[
770
+ Refused rather than half-done. A playtest screenshot needs two sessions
771
+ and only the MCP server can see both, so failing here with the reason is
772
+ better than returning the editor's own idle viewport, which looks like a
773
+ valid answer to "what does the player see".
774
+ ]]
775
+ if RunService:IsRunning() and not RunService:IsEdit() then
776
+ Dispatch.fail(
777
+ "NEEDS_EDITOR_SESSION",
778
+ "A playtest screenshot is taken on the client and read in the editor session, "
779
+ .. "so it cannot be served from the playtest server alone.",
780
+ "Use the `screenshot` tool, which drives both halves. It needs the editor "
781
+ .. "session for the same place to still be connected."
782
+ )
783
+ end
784
+
785
+ local width = math.clamp(tonumber(params.width) or DEFAULT_WIDTH, MIN_WIDTH, MAX_WIDTH)
786
+ -- Before the capture, so a path that does not resolve costs no screenshot.
787
+ local region = regionFor(params)
788
+ return encode(takeScreenshot(), width, if RunService:IsEdit() then "edit" else "playtest", region)
789
+ end
790
+
791
+ --[[
792
+ Reads a content id someone else captured.
793
+
794
+ Only the editor session can do this, and only because the temporary texture
795
+ is owned by the Studio process rather than by the session that made it.
796
+ ]]
797
+ function Capture.decode(params: { [string]: any }): { [string]: any }
798
+ local contentId = params.contentId
799
+ if typeof(contentId) ~= "string" or contentId == "" then
800
+ Dispatch.fail("BAD_PARAMS", "decode needs a `contentId`.")
801
+ end
802
+
803
+ local width = math.clamp(tonumber(params.width) or DEFAULT_WIDTH, MIN_WIDTH, MAX_WIDTH)
804
+ local context = if typeof(params.context) == "string" then params.context else "playtest client"
805
+ -- Only `rect` here: a path would have to be resolved in the client's view.
806
+ local region = if typeof(params.rect) == "string" then regionFor({ rect = params.rect }) else nil
807
+ return encode(contentId :: string, width, context, region)
808
+ end
809
+
810
+ function Capture.register()
811
+ Dispatch.registerAll("capture", {
812
+ screenshot = Capture.screenshot,
813
+ playtestId = Capture.playtestId,
814
+ decode = Capture.decode,
815
+ })
816
+ end
817
+
818
+ return Capture