@iwsdk/vite-plugin-dev 1.1.0 → 1.2.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +665 -138
- package/dist/index.js.map +1 -1
- package/dist/injection/chunks/{index-CMmj0avE.js → index-yQSSsZAQ.js} +1 -1
- package/dist/injection/chunks/{injection-template-0HSd4CcE.js → injection-template-5pO4YM1n.js} +62 -62
- package/dist/injection/runtime.js +1 -1
- package/dist/managed-browser/filmstrip.d.ts +91 -0
- package/dist/managed-browser/interaction.d.ts +19 -0
- package/dist/managed-browser/launch.d.ts +3 -1
- package/dist/managed-browser/render-mode.d.ts +78 -0
- package/dist/managed-browser/session.d.ts +4 -1
- package/dist/mcp/ws-client.d.ts +10 -0
- package/dist/native-control-notice.d.ts +17 -0
- package/dist/on-demand-emulator.d.ts +47 -0
- package/package.json +8 -8
package/dist/index.js
CHANGED
|
@@ -1150,6 +1150,7 @@ const TIMED_RUNTIME_METHODS = new Set([
|
|
|
1150
1150
|
'set_select_value',
|
|
1151
1151
|
]);
|
|
1152
1152
|
const BROWSER_RUNTIME_METHODS = new Set([
|
|
1153
|
+
'browser_filmstrip',
|
|
1153
1154
|
'browser_interact',
|
|
1154
1155
|
'browser_profile',
|
|
1155
1156
|
'browser_set_muted',
|
|
@@ -2238,6 +2239,34 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
2238
2239
|
additionalProperties: false,
|
|
2239
2240
|
},
|
|
2240
2241
|
},
|
|
2242
|
+
{
|
|
2243
|
+
name: 'browser_filmstrip',
|
|
2244
|
+
description: 'Capture a frame-accurate sequence of the application: pause the ECS, capture the current frame, then step `every` simulation frames and capture again until `count` frames are taken, and return one contact-sheet image (screenshotPath) with each panel labeled by its frame offset. Restores the previous pause state. Use it to see motion (physics, animation, locomotion) independently of how fast the browser renders.',
|
|
2245
|
+
inputSchema: {
|
|
2246
|
+
type: 'object',
|
|
2247
|
+
properties: {
|
|
2248
|
+
count: {
|
|
2249
|
+
type: 'integer',
|
|
2250
|
+
minimum: 2,
|
|
2251
|
+
maximum: 12,
|
|
2252
|
+
description: 'Frames to capture, including the current one. Defaults to 6.',
|
|
2253
|
+
},
|
|
2254
|
+
every: {
|
|
2255
|
+
type: 'integer',
|
|
2256
|
+
minimum: 1,
|
|
2257
|
+
maximum: 120,
|
|
2258
|
+
description: 'Simulation frames to step between captures (fixed 1/72 s timestep). Defaults to 5.',
|
|
2259
|
+
},
|
|
2260
|
+
format: {
|
|
2261
|
+
type: 'string',
|
|
2262
|
+
enum: ['png', 'jpeg'],
|
|
2263
|
+
description: 'Image format of the contact sheet. Defaults to png.',
|
|
2264
|
+
},
|
|
2265
|
+
},
|
|
2266
|
+
required: [],
|
|
2267
|
+
additionalProperties: false,
|
|
2268
|
+
},
|
|
2269
|
+
},
|
|
2241
2270
|
{
|
|
2242
2271
|
name: 'browser_snapshot',
|
|
2243
2272
|
description: 'Inspect the current application as a compact accessibility-first list of interactive elements and canvas regions. Use returned refs with browser_interact.',
|
|
@@ -4262,6 +4291,7 @@ const RUNTIME_TOOL_TO_METHOD = {
|
|
|
4262
4291
|
xr_cancel_sequence: 'cancel_sequence',
|
|
4263
4292
|
xr_list_targets: 'list_targets',
|
|
4264
4293
|
browser_screenshot: 'screenshot',
|
|
4294
|
+
browser_filmstrip: 'browser_filmstrip',
|
|
4265
4295
|
browser_snapshot: 'browser_snapshot',
|
|
4266
4296
|
browser_interact: 'browser_interact',
|
|
4267
4297
|
browser_profile: 'browser_profile',
|
|
@@ -4287,6 +4317,7 @@ const ALL_RUNTIME_CLI_PATHS = {
|
|
|
4287
4317
|
xr_get_gamepad_state: ['xr', 'get-gamepad-state'],
|
|
4288
4318
|
xr_set_gamepad_state: ['xr', 'set-gamepad-state'],
|
|
4289
4319
|
browser_screenshot: ['browser', 'screenshot'],
|
|
4320
|
+
browser_filmstrip: ['browser', 'filmstrip'],
|
|
4290
4321
|
browser_snapshot: ['browser', 'snapshot'],
|
|
4291
4322
|
browser_interact: ['browser', 'interact'],
|
|
4292
4323
|
browser_profile: ['browser', 'profile'],
|
|
@@ -4400,6 +4431,7 @@ const EDITOR_TARGET_MCP_TOOL_NAME_SET = new Set([
|
|
|
4400
4431
|
]);
|
|
4401
4432
|
const HOST_BROWSER_MCP_TOOL_NAME_SET = new Set([
|
|
4402
4433
|
'browser_screenshot',
|
|
4434
|
+
'browser_filmstrip',
|
|
4403
4435
|
'browser_snapshot',
|
|
4404
4436
|
'browser_interact',
|
|
4405
4437
|
'browser_profile',
|
|
@@ -22720,6 +22752,382 @@ class ManagedBrowserConsole {
|
|
|
22720
22752
|
}
|
|
22721
22753
|
}
|
|
22722
22754
|
|
|
22755
|
+
/**
|
|
22756
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
22757
|
+
*
|
|
22758
|
+
* This source code is licensed under the MIT license found in the
|
|
22759
|
+
* LICENSE file in the root directory of this source tree.
|
|
22760
|
+
*/
|
|
22761
|
+
const FILMSTRIP_DEFAULT_COUNT = 6;
|
|
22762
|
+
const FILMSTRIP_DEFAULT_EVERY = 5;
|
|
22763
|
+
const FILMSTRIP_MAX_COUNT = 12;
|
|
22764
|
+
const FILMSTRIP_MAX_EVERY = 120;
|
|
22765
|
+
const FILMSTRIP_MAX_COLUMNS = 4;
|
|
22766
|
+
function normalizeFilmstripOptions(options = {}) {
|
|
22767
|
+
const integer = (name, value, fallback, min, max) => {
|
|
22768
|
+
if (value == null) {
|
|
22769
|
+
return fallback;
|
|
22770
|
+
}
|
|
22771
|
+
if (typeof value !== 'number' ||
|
|
22772
|
+
!Number.isInteger(value) ||
|
|
22773
|
+
value < min ||
|
|
22774
|
+
value > max) {
|
|
22775
|
+
throw Object.assign(new Error(`browser_filmstrip ${name} must be an integer from ${min} to ${max}`), { retryable: false });
|
|
22776
|
+
}
|
|
22777
|
+
return value;
|
|
22778
|
+
};
|
|
22779
|
+
const format = options.format ?? 'png';
|
|
22780
|
+
if (format !== 'png' && format !== 'jpeg') {
|
|
22781
|
+
throw Object.assign(new Error('browser_filmstrip format must be png or jpeg'), { retryable: false });
|
|
22782
|
+
}
|
|
22783
|
+
return {
|
|
22784
|
+
count: integer('count', options.count, FILMSTRIP_DEFAULT_COUNT, 2, FILMSTRIP_MAX_COUNT),
|
|
22785
|
+
every: integer('every', options.every, FILMSTRIP_DEFAULT_EVERY, 1, FILMSTRIP_MAX_EVERY),
|
|
22786
|
+
format,
|
|
22787
|
+
};
|
|
22788
|
+
}
|
|
22789
|
+
/**
|
|
22790
|
+
* `ecs_step` advances one simulation step per animation frame, and the managed
|
|
22791
|
+
* browser animates at most 60 frames per second, so a filmstrip needs at least
|
|
22792
|
+
* this long however fast the app renders.
|
|
22793
|
+
*/
|
|
22794
|
+
function filmstripMinimumDurationMs(options) {
|
|
22795
|
+
return Math.ceil((((options.count - 1) * options.every) / 60) * 1000);
|
|
22796
|
+
}
|
|
22797
|
+
function filmstripOutOfTime(detail) {
|
|
22798
|
+
return Object.assign(new Error(`browser_filmstrip ran out of active command time before it could continue (${detail})`), { retryable: false });
|
|
22799
|
+
}
|
|
22800
|
+
/** Settle with `work`, or reject once `budgetMs` passes. */
|
|
22801
|
+
async function withinBudget(work, budgetMs, detail) {
|
|
22802
|
+
let timer;
|
|
22803
|
+
try {
|
|
22804
|
+
return await Promise.race([
|
|
22805
|
+
work,
|
|
22806
|
+
new Promise((_, reject) => {
|
|
22807
|
+
timer = setTimeout(() => reject(filmstripOutOfTime(detail)), Math.max(0, budgetMs));
|
|
22808
|
+
}),
|
|
22809
|
+
]);
|
|
22810
|
+
}
|
|
22811
|
+
finally {
|
|
22812
|
+
clearTimeout(timer);
|
|
22813
|
+
}
|
|
22814
|
+
}
|
|
22815
|
+
/**
|
|
22816
|
+
* Pause, capture the current frame, then step and capture until `count`
|
|
22817
|
+
* frames are taken. The caller's pause state is restored however this ends:
|
|
22818
|
+
* ECS resumes only if this call paused it.
|
|
22819
|
+
*
|
|
22820
|
+
* Every step and capture runs against the remaining command budget, so a
|
|
22821
|
+
* sequence that cannot finish fails with a clear error before the managed
|
|
22822
|
+
* browser's command coordinator would time the whole browser out.
|
|
22823
|
+
*/
|
|
22824
|
+
async function runFilmstrip(ecs, options, capture, budget) {
|
|
22825
|
+
const minimumMs = filmstripMinimumDurationMs(options);
|
|
22826
|
+
if (minimumMs > budget()) {
|
|
22827
|
+
throw Object.assign(new Error(`browser_filmstrip needs at least ${(minimumMs / 1000).toFixed(1)} s ` +
|
|
22828
|
+
`to step ${(options.count - 1) * options.every} frames at 60 fps, more than one ` +
|
|
22829
|
+
'command allows. Lower count or every.'), { retryable: false });
|
|
22830
|
+
}
|
|
22831
|
+
const wasPaused = await ecs.isPaused();
|
|
22832
|
+
if (wasPaused === null) {
|
|
22833
|
+
// Guessing would risk resuming an ECS the user paused on purpose.
|
|
22834
|
+
throw Object.assign(new Error('browser_filmstrip needs an @iwsdk/core that reports ECS pause state ' +
|
|
22835
|
+
'(MCPRuntime.ecsPaused). Update @iwsdk/core to match @iwsdk/vite-plugin-dev.'), { retryable: false });
|
|
22836
|
+
}
|
|
22837
|
+
// Set before pausing, so a pause that takes effect and then rejects is
|
|
22838
|
+
// still undone.
|
|
22839
|
+
const pausedHere = !wasPaused;
|
|
22840
|
+
let failure;
|
|
22841
|
+
try {
|
|
22842
|
+
if (pausedHere) {
|
|
22843
|
+
// Read the budget first: it throws once spent, and must do so before
|
|
22844
|
+
// the next ECS command starts.
|
|
22845
|
+
const pauseBudgetMs = budget();
|
|
22846
|
+
await withinBudget(ecs.pause(), pauseBudgetMs, 'pausing ECS');
|
|
22847
|
+
}
|
|
22848
|
+
const frames = [];
|
|
22849
|
+
for (let index = 0; index < options.count; index++) {
|
|
22850
|
+
const offset = index * options.every;
|
|
22851
|
+
if (index > 0) {
|
|
22852
|
+
const stepBudgetMs = budget();
|
|
22853
|
+
await withinBudget(ecs.step(options.every), stepBudgetMs, `stepping to frame +${offset}`);
|
|
22854
|
+
}
|
|
22855
|
+
frames.push({ offset, capture: await capture(offset) });
|
|
22856
|
+
}
|
|
22857
|
+
return frames;
|
|
22858
|
+
}
|
|
22859
|
+
catch (error) {
|
|
22860
|
+
failure = error;
|
|
22861
|
+
throw error;
|
|
22862
|
+
}
|
|
22863
|
+
finally {
|
|
22864
|
+
if (pausedHere) {
|
|
22865
|
+
try {
|
|
22866
|
+
await ecs.resume();
|
|
22867
|
+
}
|
|
22868
|
+
catch (resumeError) {
|
|
22869
|
+
if (failure === undefined) {
|
|
22870
|
+
throw Object.assign(new Error(`browser_filmstrip captured every frame but could not resume ECS: ${errorText(resumeError)}`), { cause: resumeError, retryable: false });
|
|
22871
|
+
}
|
|
22872
|
+
// Keep the original failure; record that ECS may still be paused.
|
|
22873
|
+
if (failure instanceof Error) {
|
|
22874
|
+
failure.message += ` (ECS may still be paused: resume failed with ${errorText(resumeError)})`;
|
|
22875
|
+
failure.resumeError =
|
|
22876
|
+
resumeError;
|
|
22877
|
+
}
|
|
22878
|
+
}
|
|
22879
|
+
}
|
|
22880
|
+
}
|
|
22881
|
+
}
|
|
22882
|
+
function errorText(error) {
|
|
22883
|
+
return error instanceof Error ? error.message : String(error);
|
|
22884
|
+
}
|
|
22885
|
+
/** ECS controls through the page runtime's MCP dispatch. */
|
|
22886
|
+
function pageFilmstripEcsControl(frame) {
|
|
22887
|
+
const dispatch = (method, params = {}) => frame.evaluate(async ({ method, params }) => {
|
|
22888
|
+
const runtime = window.FRAMEWORK_MCP_RUNTIME;
|
|
22889
|
+
if (typeof runtime?.dispatch !== 'function') {
|
|
22890
|
+
throw new Error('browser_filmstrip needs an IWSDK app runtime (FRAMEWORK_MCP_RUNTIME)');
|
|
22891
|
+
}
|
|
22892
|
+
return runtime.dispatch(method, params);
|
|
22893
|
+
}, { method, params });
|
|
22894
|
+
return {
|
|
22895
|
+
isPaused: () => frame.evaluate(() => {
|
|
22896
|
+
const paused = window.FRAMEWORK_MCP_RUNTIME?.ecsPaused;
|
|
22897
|
+
return typeof paused === 'boolean' ? paused : null;
|
|
22898
|
+
}),
|
|
22899
|
+
pause: async () => {
|
|
22900
|
+
await dispatch('ecs_pause');
|
|
22901
|
+
},
|
|
22902
|
+
step: async (frames) => {
|
|
22903
|
+
await dispatch('ecs_step', { count: frames });
|
|
22904
|
+
},
|
|
22905
|
+
resume: async () => {
|
|
22906
|
+
await dispatch('ecs_resume');
|
|
22907
|
+
},
|
|
22908
|
+
};
|
|
22909
|
+
}
|
|
22910
|
+
/**
|
|
22911
|
+
* Lay `count` panels of `source` aspect out in at most four columns, balanced
|
|
22912
|
+
* across rows (6 panels make 3×2, not 4+2), scaled down so the whole sheet
|
|
22913
|
+
* fits `bounds`.
|
|
22914
|
+
*/
|
|
22915
|
+
function contactSheetLayout(count, source, bounds) {
|
|
22916
|
+
const rows = Math.ceil(count / FILMSTRIP_MAX_COLUMNS);
|
|
22917
|
+
const columns = Math.ceil(count / rows);
|
|
22918
|
+
const scale = Math.min(1, bounds.width / (columns * source.width), bounds.height / (rows * source.height));
|
|
22919
|
+
const panelWidth = Math.max(1, Math.floor(source.width * scale));
|
|
22920
|
+
const panelHeight = Math.max(1, Math.floor(source.height * scale));
|
|
22921
|
+
return {
|
|
22922
|
+
columns,
|
|
22923
|
+
rows,
|
|
22924
|
+
panelWidth,
|
|
22925
|
+
panelHeight,
|
|
22926
|
+
width: columns * panelWidth,
|
|
22927
|
+
height: rows * panelHeight,
|
|
22928
|
+
};
|
|
22929
|
+
}
|
|
22930
|
+
function offsetLabel(offset, panelHeight) {
|
|
22931
|
+
const fontSize = Math.max(12, Math.round(panelHeight * 0.06));
|
|
22932
|
+
const padding = Math.round(fontSize * 0.45);
|
|
22933
|
+
const text = `+${offset}`;
|
|
22934
|
+
const width = Math.round(text.length * fontSize * 0.62 + padding * 2);
|
|
22935
|
+
const height = Math.round(fontSize + padding * 1.6);
|
|
22936
|
+
return Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">` +
|
|
22937
|
+
`<rect width="${width}" height="${height}" rx="${Math.round(padding * 0.8)}" fill="#000000" fill-opacity="0.72"/>` +
|
|
22938
|
+
`<text x="${padding}" y="${Math.round(padding * 0.8 + fontSize * 0.85)}" font-family="DejaVu Sans Mono, monospace" font-size="${fontSize}" font-weight="700" fill="#ffffff">${text}</text>` +
|
|
22939
|
+
`</svg>`);
|
|
22940
|
+
}
|
|
22941
|
+
/**
|
|
22942
|
+
* Stitch captures into one contact sheet, each panel labeled with its
|
|
22943
|
+
* simulation frame offset (`+0`, `+5`, ...).
|
|
22944
|
+
*/
|
|
22945
|
+
async function composeContactSheet(panels, bounds, format) {
|
|
22946
|
+
const first = await sharp(panels[0].bytes).metadata();
|
|
22947
|
+
const layout = contactSheetLayout(panels.length, { width: first.width ?? 1, height: first.height ?? 1 }, bounds);
|
|
22948
|
+
const layers = await Promise.all(panels.map(async ({ offset, bytes }, index) => {
|
|
22949
|
+
const left = (index % layout.columns) * layout.panelWidth;
|
|
22950
|
+
const top = Math.floor(index / layout.columns) * layout.panelHeight;
|
|
22951
|
+
const resized = await sharp(bytes)
|
|
22952
|
+
.resize(layout.panelWidth, layout.panelHeight, { fit: 'fill' })
|
|
22953
|
+
.png()
|
|
22954
|
+
.toBuffer();
|
|
22955
|
+
const margin = Math.max(4, Math.round(layout.panelHeight * 0.02));
|
|
22956
|
+
return [
|
|
22957
|
+
{ input: resized, left, top },
|
|
22958
|
+
{
|
|
22959
|
+
input: offsetLabel(offset, layout.panelHeight),
|
|
22960
|
+
left: left + margin,
|
|
22961
|
+
top: top + margin,
|
|
22962
|
+
},
|
|
22963
|
+
];
|
|
22964
|
+
}));
|
|
22965
|
+
const sheet = sharp({
|
|
22966
|
+
create: {
|
|
22967
|
+
background: { alpha: 1, b: 0, g: 0, r: 0 },
|
|
22968
|
+
channels: 4,
|
|
22969
|
+
height: layout.height,
|
|
22970
|
+
width: layout.width,
|
|
22971
|
+
},
|
|
22972
|
+
}).composite(layers.flat());
|
|
22973
|
+
const bytes = await (format === 'jpeg'
|
|
22974
|
+
? sheet.jpeg({ quality: 90 }).toBuffer()
|
|
22975
|
+
: sheet.png().toBuffer());
|
|
22976
|
+
return { bytes, layout };
|
|
22977
|
+
}
|
|
22978
|
+
|
|
22979
|
+
/**
|
|
22980
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
22981
|
+
*
|
|
22982
|
+
* This source code is licensed under the MIT license found in the
|
|
22983
|
+
* LICENSE file in the root directory of this source tree.
|
|
22984
|
+
*/
|
|
22985
|
+
const warnedRenderModeOverrides = new Set();
|
|
22986
|
+
/**
|
|
22987
|
+
* Resolve the render mode for a managed browser session. Headed sessions are
|
|
22988
|
+
* always continuous because a person is watching them. `IWSDK_DEV_RENDER_MODE`
|
|
22989
|
+
* (`continuous` or `on-demand`, case-insensitive) overrides the headless
|
|
22990
|
+
* default; any other value is reported once and ignored.
|
|
22991
|
+
*/
|
|
22992
|
+
function resolveManagedRenderMode(headless, override = process.env.IWSDK_DEV_RENDER_MODE, warn = console.warn) {
|
|
22993
|
+
if (!headless) {
|
|
22994
|
+
return 'continuous';
|
|
22995
|
+
}
|
|
22996
|
+
const normalized = override?.trim().toLowerCase();
|
|
22997
|
+
if (!normalized) {
|
|
22998
|
+
return 'on-demand';
|
|
22999
|
+
}
|
|
23000
|
+
if (normalized === 'continuous' || normalized === 'on-demand') {
|
|
23001
|
+
return normalized;
|
|
23002
|
+
}
|
|
23003
|
+
if (!warnedRenderModeOverrides.has(normalized)) {
|
|
23004
|
+
warnedRenderModeOverrides.add(normalized);
|
|
23005
|
+
warn(`[IWSDK Dev] Ignoring IWSDK_DEV_RENDER_MODE=${JSON.stringify(override)}: expected "continuous" or "on-demand". Headless sessions render on demand.`);
|
|
23006
|
+
}
|
|
23007
|
+
return 'on-demand';
|
|
23008
|
+
}
|
|
23009
|
+
/**
|
|
23010
|
+
* Draw `frames` fresh frames in the application frame and wait until they have
|
|
23011
|
+
* been handed to the compositor. Apps that do not expose `requestRender`
|
|
23012
|
+
* (non-IWSDK runtimes, older @iwsdk/core) are left alone.
|
|
23013
|
+
*/
|
|
23014
|
+
async function requestApplicationRender(frame, frames = 1, timeoutMs = 10000) {
|
|
23015
|
+
await frame.evaluate(({ frames, timeoutMs }) => {
|
|
23016
|
+
const runtime = window.FRAMEWORK_MCP_RUNTIME;
|
|
23017
|
+
if (typeof runtime?.requestRender !== 'function') {
|
|
23018
|
+
return;
|
|
23019
|
+
}
|
|
23020
|
+
return Promise.race([
|
|
23021
|
+
runtime.requestRender(frames),
|
|
23022
|
+
new Promise((resolve) => setTimeout(resolve, timeoutMs)),
|
|
23023
|
+
]);
|
|
23024
|
+
}, { frames, timeoutMs });
|
|
23025
|
+
}
|
|
23026
|
+
/**
|
|
23027
|
+
* Whether a frame the application draws stays on its canvas until the next
|
|
23028
|
+
* draw. False under the emulator's default per-frame clear in immersive
|
|
23029
|
+
* sessions; the injected runtime turns that clear off for on-demand sessions
|
|
23030
|
+
* and sets `__IWSDK_KEEPS_DRAWN_FRAMES`.
|
|
23031
|
+
*/
|
|
23032
|
+
async function applicationKeepsDrawnFrames(frame) {
|
|
23033
|
+
return frame
|
|
23034
|
+
.evaluate(() => window.__IWSDK_KEEPS_DRAWN_FRAMES === true)
|
|
23035
|
+
.catch(() => false);
|
|
23036
|
+
}
|
|
23037
|
+
/**
|
|
23038
|
+
* Start drawing every frame in the application until the returned function is
|
|
23039
|
+
* called. Holds are counted by the runtime, so overlapping holds (a capture
|
|
23040
|
+
* during a profile) cannot end each other early. Returns `null` when the app
|
|
23041
|
+
* does not support render holds (non-IWSDK runtimes, older @iwsdk/core); such
|
|
23042
|
+
* apps draw continuously anyway.
|
|
23043
|
+
*/
|
|
23044
|
+
async function acquireApplicationRenderHold(frame) {
|
|
23045
|
+
return frame.evaluate(() => {
|
|
23046
|
+
const runtime = window.FRAMEWORK_MCP_RUNTIME;
|
|
23047
|
+
return typeof runtime?.acquireRenderHold === 'function'
|
|
23048
|
+
? runtime.acquireRenderHold()
|
|
23049
|
+
: null;
|
|
23050
|
+
});
|
|
23051
|
+
}
|
|
23052
|
+
/**
|
|
23053
|
+
* Release a hold from {@link acquireApplicationRenderHold}. A reload drops
|
|
23054
|
+
* every hold with the page, so a missing runtime is not an error.
|
|
23055
|
+
*/
|
|
23056
|
+
async function releaseApplicationRenderHold(frame, id) {
|
|
23057
|
+
if (id == null) {
|
|
23058
|
+
return;
|
|
23059
|
+
}
|
|
23060
|
+
await frame
|
|
23061
|
+
.evaluate((id) => {
|
|
23062
|
+
window.FRAMEWORK_MCP_RUNTIME?.releaseRenderHold?.(id);
|
|
23063
|
+
}, id)
|
|
23064
|
+
.catch(() => { });
|
|
23065
|
+
}
|
|
23066
|
+
/**
|
|
23067
|
+
* Run `fn` with the application drawing every frame. Profiling, runtime
|
|
23068
|
+
* evidence and pixel captures under the emulator's per-frame clear need real
|
|
23069
|
+
* frames, which on-demand rendering would otherwise skip.
|
|
23070
|
+
*/
|
|
23071
|
+
async function withApplicationRenderHold(frame, fn) {
|
|
23072
|
+
const id = await acquireApplicationRenderHold(frame);
|
|
23073
|
+
try {
|
|
23074
|
+
return await fn();
|
|
23075
|
+
}
|
|
23076
|
+
finally {
|
|
23077
|
+
await releaseApplicationRenderHold(frame, id);
|
|
23078
|
+
}
|
|
23079
|
+
}
|
|
23080
|
+
/** Longest a capture waits for its fresh frame before capturing anyway. */
|
|
23081
|
+
const MAX_RENDER_REQUEST_MS = 10000;
|
|
23082
|
+
/**
|
|
23083
|
+
* Capture pixels from an application that may render on demand. When drawn
|
|
23084
|
+
* frames persist, one fresh frame is requested and captured. Otherwise (the
|
|
23085
|
+
* emulator clears the canvas every immersive frame) rendering is held for the
|
|
23086
|
+
* whole capture so the frame on screen is always current. The render request
|
|
23087
|
+
* gets a bounded share of `timeoutMs`; `capture` receives what is left.
|
|
23088
|
+
*/
|
|
23089
|
+
async function captureFreshApplicationFrame(frame, timeoutMs, capture) {
|
|
23090
|
+
const startedAt = Date.now();
|
|
23091
|
+
const renderBudgetMs = Math.max(1, Math.min(timeoutMs, MAX_RENDER_REQUEST_MS));
|
|
23092
|
+
const remaining = () => Math.max(1, timeoutMs - (Date.now() - startedAt));
|
|
23093
|
+
if (await applicationKeepsDrawnFrames(frame)) {
|
|
23094
|
+
await requestApplicationRender(frame, 1, renderBudgetMs);
|
|
23095
|
+
return capture(remaining());
|
|
23096
|
+
}
|
|
23097
|
+
return withApplicationRenderHold(frame, async () => {
|
|
23098
|
+
await requestApplicationRender(frame, 1, renderBudgetMs);
|
|
23099
|
+
return capture(remaining());
|
|
23100
|
+
});
|
|
23101
|
+
}
|
|
23102
|
+
/**
|
|
23103
|
+
* Create a {@link RenderHoldTracker}. Evidence collection waits for a
|
|
23104
|
+
* reloaded runtime to appear; a hold requested before it exists comes back
|
|
23105
|
+
* `null` and must be requested again once it does.
|
|
23106
|
+
*/
|
|
23107
|
+
function createRenderHoldTracker() {
|
|
23108
|
+
let current = null;
|
|
23109
|
+
return {
|
|
23110
|
+
async ensure(frame) {
|
|
23111
|
+
if (current?.frame === frame && current.id != null) {
|
|
23112
|
+
return;
|
|
23113
|
+
}
|
|
23114
|
+
if (current != null && current.frame !== frame) {
|
|
23115
|
+
await releaseApplicationRenderHold(current.frame, current.id);
|
|
23116
|
+
}
|
|
23117
|
+
const id = await acquireApplicationRenderHold(frame).catch(() => null);
|
|
23118
|
+
current = { frame, id };
|
|
23119
|
+
},
|
|
23120
|
+
async release() {
|
|
23121
|
+
if (current == null) {
|
|
23122
|
+
return;
|
|
23123
|
+
}
|
|
23124
|
+
const { frame, id } = current;
|
|
23125
|
+
current = null;
|
|
23126
|
+
await releaseApplicationRenderHold(frame, id);
|
|
23127
|
+
},
|
|
23128
|
+
};
|
|
23129
|
+
}
|
|
23130
|
+
|
|
22723
23131
|
/**
|
|
22724
23132
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
22725
23133
|
*
|
|
@@ -22999,6 +23407,10 @@ class ManagedInteractionSurface {
|
|
|
22999
23407
|
captureRuntimeScreenshot(options = {}) {
|
|
23000
23408
|
return withRuntimeView(this.options.page, () => this.captureApplicationImage(options, this.options.getRemainingOperationTimeMs('browser_screenshot')), (error) => this.options.onRestoreError('browser_screenshot', error));
|
|
23001
23409
|
}
|
|
23410
|
+
captureRuntimeFilmstrip(options = {}) {
|
|
23411
|
+
const normalized = normalizeFilmstripOptions(options);
|
|
23412
|
+
return withRuntimeView(this.options.page, () => this.captureApplicationFilmstrip(normalized), (error) => this.options.onRestoreError('browser_filmstrip', error));
|
|
23413
|
+
}
|
|
23002
23414
|
snapshotApplication(request = {}) {
|
|
23003
23415
|
return collectApplicationSnapshot(this.options.page, this.snapshotRecipes, request, this.options.getRemainingOperationTimeMs('browser_snapshot'));
|
|
23004
23416
|
}
|
|
@@ -23014,9 +23426,40 @@ class ManagedInteractionSurface {
|
|
|
23014
23426
|
}
|
|
23015
23427
|
return withRuntimeView(this.options.page, () => this.runInteractionBatch(request), (error) => this.options.onRestoreError('browser_interact', error));
|
|
23016
23428
|
}
|
|
23429
|
+
async captureApplicationFilmstrip(options) {
|
|
23430
|
+
const { getRemainingOperationTimeMs, page, screenshotSize } = this.options;
|
|
23431
|
+
const frame = await resolveApplicationFrame(page);
|
|
23432
|
+
// Panels are captured losslessly; only the sheet uses the requested format.
|
|
23433
|
+
// Every step and capture re-reads the active budget, so a sequence that
|
|
23434
|
+
// cannot fit fails with the budget error instead of overrunning the command.
|
|
23435
|
+
const budget = () => getRemainingOperationTimeMs('browser_filmstrip');
|
|
23436
|
+
const frames = await runFilmstrip(pageFilmstripEcsControl(frame), options, async () => (await this.captureApplicationImage({ format: 'png' }, budget())).bytes, budget);
|
|
23437
|
+
const { bytes, layout } = await composeContactSheet(frames.map(({ offset, capture }) => ({ offset, bytes: capture })), { width: screenshotSize.width * 2, height: screenshotSize.height * 2 }, options.format);
|
|
23438
|
+
return {
|
|
23439
|
+
bytes,
|
|
23440
|
+
metadata: {
|
|
23441
|
+
...(await getApplicationMetadata(page)),
|
|
23442
|
+
columns: layout.columns,
|
|
23443
|
+
count: options.count,
|
|
23444
|
+
every: options.every,
|
|
23445
|
+
frameOffsets: frames.map(({ offset }) => offset),
|
|
23446
|
+
height: layout.height,
|
|
23447
|
+
mimeType: options.format === 'jpeg' ? 'image/jpeg' : 'image/png',
|
|
23448
|
+
panelHeight: layout.panelHeight,
|
|
23449
|
+
panelWidth: layout.panelWidth,
|
|
23450
|
+
rows: layout.rows,
|
|
23451
|
+
width: layout.width,
|
|
23452
|
+
},
|
|
23453
|
+
};
|
|
23454
|
+
}
|
|
23017
23455
|
async captureApplicationImage(options, timeoutMs) {
|
|
23456
|
+
const frame = await resolveApplicationFrame(this.options.page);
|
|
23457
|
+
// An on-demand app has nothing current on its canvas: draw a fresh frame
|
|
23458
|
+
// (holding rendering where the emulator would clear it) before capturing.
|
|
23459
|
+
return captureFreshApplicationFrame(frame, timeoutMs, (remainingMs) => this.captureRenderedApplicationImage(frame, options, remainingMs));
|
|
23460
|
+
}
|
|
23461
|
+
async captureRenderedApplicationImage(frame, options, timeoutMs) {
|
|
23018
23462
|
const { page, screenshotSize } = this.options;
|
|
23019
|
-
const frame = await resolveApplicationFrame(page);
|
|
23020
23463
|
const format = options.format ?? 'png';
|
|
23021
23464
|
const quality = Math.min(100, Math.max(20, options.quality ?? 90));
|
|
23022
23465
|
const screenshotOptions = format === 'jpeg'
|
|
@@ -24454,7 +24897,7 @@ async function openManagedChromium({ browserAutomation, headless, launchUrl, man
|
|
|
24454
24897
|
throw error;
|
|
24455
24898
|
}
|
|
24456
24899
|
}
|
|
24457
|
-
async function navigateAndVerifyManagedPage({ browserAutomationEndpoint, context, managedWorkspaceToken, page, readiness, waitForRuntime = true, signal, traceMcp, url, }) {
|
|
24900
|
+
async function navigateAndVerifyManagedPage({ browserAutomationEndpoint, context, managedWorkspaceToken, page, readiness, renderMode = 'continuous', waitForRuntime = true, signal, traceMcp, url, }) {
|
|
24458
24901
|
throwIfLaunchAborted(signal);
|
|
24459
24902
|
if (managedWorkspaceToken != null) {
|
|
24460
24903
|
await page.addInitScript((queryName) => {
|
|
@@ -24479,11 +24922,12 @@ async function navigateAndVerifyManagedPage({ browserAutomationEndpoint, context
|
|
|
24479
24922
|
console.error(`[unhandledrejection] ${text}`);
|
|
24480
24923
|
});
|
|
24481
24924
|
});
|
|
24482
|
-
await page.addInitScript((traceEnabled) => {
|
|
24925
|
+
await page.addInitScript(({ renderMode, traceEnabled }) => {
|
|
24483
24926
|
window.__IWER_MCP_MANAGED = true;
|
|
24484
24927
|
window.__IWSDK_MCP_TRACE = traceEnabled;
|
|
24485
24928
|
window.__IWSDK_MANAGED_BROWSER_TARGET = true;
|
|
24486
|
-
|
|
24929
|
+
window.__IWSDK_RENDER_MODE = renderMode;
|
|
24930
|
+
}, { renderMode, traceEnabled: traceMcp });
|
|
24487
24931
|
try {
|
|
24488
24932
|
await page.goto(url, { waitUntil: 'commit' });
|
|
24489
24933
|
}
|
|
@@ -24852,6 +25296,14 @@ class BrowserProfiler {
|
|
|
24852
25296
|
}
|
|
24853
25297
|
}
|
|
24854
25298
|
async function startProfileInFrame(frame, mode) {
|
|
25299
|
+
// A profile measures real frame cost, so draw every frame while it runs.
|
|
25300
|
+
// The hold id lives on the page next to the profile state it belongs to.
|
|
25301
|
+
const renderHold = await acquireApplicationRenderHold(frame);
|
|
25302
|
+
await frame.evaluate((id) => {
|
|
25303
|
+
const runtimeWindow = window;
|
|
25304
|
+
runtimeWindow.FRAMEWORK_MCP_RUNTIME?.releaseRenderHold?.(runtimeWindow.__IWSDK_PROFILE_RENDER_HOLD);
|
|
25305
|
+
runtimeWindow.__IWSDK_PROFILE_RENDER_HOLD = id;
|
|
25306
|
+
}, renderHold);
|
|
24855
25307
|
await frame.evaluate(({ maxSamples, mode }) => {
|
|
24856
25308
|
const runtimeWindow = window;
|
|
24857
25309
|
runtimeWindow.__IWSDK_BROWSER_PROFILE?.observers?.forEach((observer) => observer.disconnect());
|
|
@@ -24925,6 +25377,16 @@ async function startProfileInFrame(frame, mode) {
|
|
|
24925
25377
|
}, { maxSamples: MAX_PROFILE_SAMPLES, mode });
|
|
24926
25378
|
}
|
|
24927
25379
|
async function stopProfileInFrame(frame) {
|
|
25380
|
+
await frame
|
|
25381
|
+
.evaluate(() => {
|
|
25382
|
+
const runtimeWindow = window;
|
|
25383
|
+
const id = runtimeWindow.__IWSDK_PROFILE_RENDER_HOLD;
|
|
25384
|
+
delete runtimeWindow.__IWSDK_PROFILE_RENDER_HOLD;
|
|
25385
|
+
if (id != null) {
|
|
25386
|
+
runtimeWindow.FRAMEWORK_MCP_RUNTIME?.releaseRenderHold?.(id);
|
|
25387
|
+
}
|
|
25388
|
+
})
|
|
25389
|
+
.catch(() => { });
|
|
24928
25390
|
const result = await frame.evaluate((maxSamples) => {
|
|
24929
25391
|
const runtimeWindow = window;
|
|
24930
25392
|
const state = runtimeWindow.__IWSDK_BROWSER_PROFILE;
|
|
@@ -25006,58 +25468,69 @@ async function collectRuntimePreflightEvidence(page, request) {
|
|
|
25006
25468
|
};
|
|
25007
25469
|
});
|
|
25008
25470
|
const frame = await resolveApplicationFrame(page);
|
|
25009
|
-
|
|
25010
|
-
|
|
25011
|
-
|
|
25012
|
-
|
|
25013
|
-
|
|
25014
|
-
|
|
25015
|
-
|
|
25016
|
-
|
|
25017
|
-
|
|
25018
|
-
|
|
25019
|
-
|
|
25020
|
-
|
|
25021
|
-
|
|
25022
|
-
|
|
25471
|
+
// Evidence reads real draw calls and frame cost, so the app draws every
|
|
25472
|
+
// frame while it is collected. The runtime may not exist yet, so the hold
|
|
25473
|
+
// is retried inside the readiness loop below.
|
|
25474
|
+
const renderHold = createRenderHoldTracker();
|
|
25475
|
+
try {
|
|
25476
|
+
const runtimeDeadline = Date.now() + 20000;
|
|
25477
|
+
let hierarchy = null;
|
|
25478
|
+
let hierarchyEntries = [];
|
|
25479
|
+
let runtimeHashes = [];
|
|
25480
|
+
let renderStats = { available: false };
|
|
25481
|
+
let lastRuntimeError = null;
|
|
25482
|
+
while (Date.now() < runtimeDeadline) {
|
|
25483
|
+
try {
|
|
25484
|
+
await renderHold.ensure(frame);
|
|
25485
|
+
await requestApplicationRender(frame, 1, 5000);
|
|
25486
|
+
hierarchy = await frame.evaluate(readRuntimeHierarchy);
|
|
25487
|
+
({ entries: hierarchyEntries, runtimeHashes } =
|
|
25488
|
+
flattenRuntimeHierarchy(hierarchy));
|
|
25489
|
+
renderStats = await frame.evaluate(readRuntimeRenderStats);
|
|
25490
|
+
if (runtimeHashes.length > 0) {
|
|
25491
|
+
break;
|
|
25492
|
+
}
|
|
25493
|
+
}
|
|
25494
|
+
catch (error) {
|
|
25495
|
+
lastRuntimeError = error;
|
|
25023
25496
|
}
|
|
25497
|
+
await page.waitForTimeout(50);
|
|
25024
25498
|
}
|
|
25025
|
-
|
|
25026
|
-
lastRuntimeError
|
|
25499
|
+
if (hierarchy == null) {
|
|
25500
|
+
throw new Error(`IWSDK app runtime bridge is unavailable: ${errorMessage$1(lastRuntimeError ?? 'timed out')}`);
|
|
25027
25501
|
}
|
|
25028
|
-
await
|
|
25502
|
+
const frameTimes = await frame.evaluate(sampleRuntimeFrameTimes, {
|
|
25503
|
+
sampleFrames: request.sampleFrames,
|
|
25504
|
+
warmupFrames: request.warmupFrames,
|
|
25505
|
+
});
|
|
25506
|
+
const environmentRead = await frame.evaluate(readEnvironmentDescriptor, 'renderer');
|
|
25507
|
+
assertBrowserEnvironmentDescriptor(environmentRead.environment);
|
|
25508
|
+
const presentation = await collectPresentationSnapshot(frame, {
|
|
25509
|
+
frameTimes,
|
|
25510
|
+
hierarchyObjectCount: hierarchyEntries.length,
|
|
25511
|
+
renderStats,
|
|
25512
|
+
warmupFrames: request.warmupFrames,
|
|
25513
|
+
});
|
|
25514
|
+
return {
|
|
25515
|
+
...presentation,
|
|
25516
|
+
collectedAt: Date.now(),
|
|
25517
|
+
editor,
|
|
25518
|
+
environment: {
|
|
25519
|
+
...environmentRead.environment,
|
|
25520
|
+
canvas: environmentRead.canvas == null
|
|
25521
|
+
? null
|
|
25522
|
+
: {
|
|
25523
|
+
height: environmentRead.canvas.backingHeight,
|
|
25524
|
+
width: environmentRead.canvas.backingWidth,
|
|
25525
|
+
},
|
|
25526
|
+
},
|
|
25527
|
+
renderStats,
|
|
25528
|
+
runtimeHashes,
|
|
25529
|
+
};
|
|
25029
25530
|
}
|
|
25030
|
-
|
|
25031
|
-
|
|
25531
|
+
finally {
|
|
25532
|
+
await renderHold.release();
|
|
25032
25533
|
}
|
|
25033
|
-
const frameTimes = await frame.evaluate(sampleRuntimeFrameTimes, {
|
|
25034
|
-
sampleFrames: request.sampleFrames,
|
|
25035
|
-
warmupFrames: request.warmupFrames,
|
|
25036
|
-
});
|
|
25037
|
-
const environmentRead = await frame.evaluate(readEnvironmentDescriptor, 'renderer');
|
|
25038
|
-
assertBrowserEnvironmentDescriptor(environmentRead.environment);
|
|
25039
|
-
const presentation = await collectPresentationSnapshot(frame, {
|
|
25040
|
-
frameTimes,
|
|
25041
|
-
hierarchyObjectCount: hierarchyEntries.length,
|
|
25042
|
-
renderStats,
|
|
25043
|
-
warmupFrames: request.warmupFrames,
|
|
25044
|
-
});
|
|
25045
|
-
return {
|
|
25046
|
-
...presentation,
|
|
25047
|
-
collectedAt: Date.now(),
|
|
25048
|
-
editor,
|
|
25049
|
-
environment: {
|
|
25050
|
-
...environmentRead.environment,
|
|
25051
|
-
canvas: environmentRead.canvas == null
|
|
25052
|
-
? null
|
|
25053
|
-
: {
|
|
25054
|
-
height: environmentRead.canvas.backingHeight,
|
|
25055
|
-
width: environmentRead.canvas.backingWidth,
|
|
25056
|
-
},
|
|
25057
|
-
},
|
|
25058
|
-
renderStats,
|
|
25059
|
-
runtimeHashes,
|
|
25060
|
-
};
|
|
25061
25534
|
});
|
|
25062
25535
|
}
|
|
25063
25536
|
async function collectRuntimePublishEvidence(page, request, managedWorkspaceToken = null) {
|
|
@@ -25241,91 +25714,103 @@ async function collectRuntimePublishEvidence(page, request, managedWorkspaceToke
|
|
|
25241
25714
|
frame.src = url.pathname + url.search + url.hash;
|
|
25242
25715
|
}, { source: runtimeSource, token: reloadToken });
|
|
25243
25716
|
await page.waitForTimeout(250);
|
|
25244
|
-
|
|
25245
|
-
|
|
25246
|
-
|
|
25247
|
-
|
|
25248
|
-
const
|
|
25249
|
-
|
|
25250
|
-
|
|
25251
|
-
|
|
25252
|
-
|
|
25253
|
-
|
|
25254
|
-
|
|
25255
|
-
|
|
25256
|
-
|
|
25257
|
-
|
|
25258
|
-
|
|
25259
|
-
|
|
25260
|
-
|
|
25261
|
-
|
|
25262
|
-
|
|
25263
|
-
|
|
25264
|
-
|
|
25265
|
-
const
|
|
25266
|
-
|
|
25267
|
-
|
|
25268
|
-
|
|
25269
|
-
|
|
25270
|
-
|
|
25271
|
-
|
|
25272
|
-
|
|
25273
|
-
|
|
25274
|
-
|
|
25275
|
-
|
|
25276
|
-
|
|
25717
|
+
// Evidence reads real draw calls and frame cost, so the app draws every
|
|
25718
|
+
// frame while it is collected. The reload drops a hold with its page, and
|
|
25719
|
+
// the reloaded runtime may not exist on the first attempts, so the hold
|
|
25720
|
+
// is re-taken on each candidate frame until it sticks.
|
|
25721
|
+
const renderHold = createRenderHoldTracker();
|
|
25722
|
+
try {
|
|
25723
|
+
let live = null;
|
|
25724
|
+
let runtimeFrame = null;
|
|
25725
|
+
let lastLiveError = null;
|
|
25726
|
+
let runtimeObservedAt = null;
|
|
25727
|
+
const liveDeadline = Date.now() + 20000;
|
|
25728
|
+
while (Date.now() < liveDeadline) {
|
|
25729
|
+
try {
|
|
25730
|
+
const candidateFrame = await resolveApplicationFrame(page);
|
|
25731
|
+
if (new URL(candidateFrame.url()).searchParams.get('__iwsdk_publish_reload') !== reloadToken) {
|
|
25732
|
+
throw new Error('App runtime reload is still in progress');
|
|
25733
|
+
}
|
|
25734
|
+
await renderHold.ensure(candidateFrame);
|
|
25735
|
+
await requestApplicationRender(candidateFrame, 1, 5000);
|
|
25736
|
+
const hierarchy = await candidateFrame.evaluate(readRuntimeHierarchy);
|
|
25737
|
+
const { entries, runtimeHashes } = flattenRuntimeHierarchy(hierarchy);
|
|
25738
|
+
const nodes = await candidateFrame.evaluate(readRuntimeNodes, request.nodeIds.map((nodeId) => ({
|
|
25739
|
+
entry: entries.find((candidate) => candidate.sceneNodeId === nodeId) ??
|
|
25740
|
+
null,
|
|
25741
|
+
nodeId,
|
|
25742
|
+
})));
|
|
25743
|
+
const renderStats = await candidateFrame.evaluate(readRuntimeRenderStats);
|
|
25744
|
+
if (runtimeHashes.length > 0) {
|
|
25745
|
+
runtimeObservedAt ?? (runtimeObservedAt = Date.now());
|
|
25746
|
+
const allNodesPresent = nodes.every((entry) => entry.hierarchy != null);
|
|
25747
|
+
if (allNodesPresent || Date.now() - runtimeObservedAt >= 3000) {
|
|
25748
|
+
const frameTimeSamplesMs = await candidateFrame.evaluate(sampleRuntimeFrameTimes, { sampleFrames: 8, warmupFrames: 2 });
|
|
25749
|
+
live = {
|
|
25750
|
+
hierarchy,
|
|
25751
|
+
hierarchyObjectCount: entries.length,
|
|
25752
|
+
nodes,
|
|
25753
|
+
renderStats: { ...renderStats, frameTimeSamplesMs },
|
|
25754
|
+
runtimeHashes,
|
|
25755
|
+
};
|
|
25756
|
+
runtimeFrame = candidateFrame;
|
|
25757
|
+
break;
|
|
25758
|
+
}
|
|
25277
25759
|
}
|
|
25278
25760
|
}
|
|
25279
|
-
|
|
25280
|
-
|
|
25281
|
-
|
|
25282
|
-
|
|
25283
|
-
|
|
25284
|
-
|
|
25285
|
-
|
|
25286
|
-
|
|
25287
|
-
|
|
25288
|
-
|
|
25289
|
-
|
|
25290
|
-
|
|
25291
|
-
|
|
25292
|
-
|
|
25293
|
-
|
|
25294
|
-
|
|
25295
|
-
|
|
25296
|
-
|
|
25297
|
-
|
|
25298
|
-
|
|
25299
|
-
|
|
25300
|
-
|
|
25301
|
-
|
|
25302
|
-
|
|
25303
|
-
|
|
25304
|
-
|
|
25305
|
-
|
|
25306
|
-
|
|
25307
|
-
|
|
25308
|
-
|
|
25309
|
-
|
|
25310
|
-
|
|
25311
|
-
|
|
25312
|
-
|
|
25313
|
-
|
|
25314
|
-
|
|
25315
|
-
|
|
25316
|
-
|
|
25317
|
-
|
|
25318
|
-
|
|
25319
|
-
|
|
25320
|
-
|
|
25321
|
-
|
|
25322
|
-
|
|
25323
|
-
|
|
25324
|
-
|
|
25325
|
-
|
|
25326
|
-
|
|
25327
|
-
|
|
25328
|
-
}
|
|
25761
|
+
catch (error) {
|
|
25762
|
+
lastLiveError = error;
|
|
25763
|
+
}
|
|
25764
|
+
await page.waitForTimeout(100);
|
|
25765
|
+
}
|
|
25766
|
+
if (live == null || runtimeFrame == null) {
|
|
25767
|
+
throw new Error(`App runtime did not become publish-ready: ${errorMessage$1(lastLiveError ?? 'timed out')}`);
|
|
25768
|
+
}
|
|
25769
|
+
const environmentRead = await runtimeFrame.evaluate(readEnvironmentDescriptor, 'largest');
|
|
25770
|
+
assertBrowserEnvironmentDescriptor(environmentRead.environment);
|
|
25771
|
+
if (environmentRead.canvas == null ||
|
|
25772
|
+
environmentRead.canvasIndex == null ||
|
|
25773
|
+
environmentRead.canvasIndex < 0) {
|
|
25774
|
+
throw new Error('App runtime did not render a canvas');
|
|
25775
|
+
}
|
|
25776
|
+
const canvases = runtimeFrame.locator('canvas');
|
|
25777
|
+
const canvasCount = await canvases.count();
|
|
25778
|
+
if (environmentRead.canvasIndex >= canvasCount) {
|
|
25779
|
+
throw new Error('App runtime frame has no canvas to capture');
|
|
25780
|
+
}
|
|
25781
|
+
const captureBytes = Buffer.from(await canvases
|
|
25782
|
+
.nth(environmentRead.canvasIndex)
|
|
25783
|
+
.screenshot({ type: 'png' }));
|
|
25784
|
+
const capture = await inspectPngCapture(captureBytes);
|
|
25785
|
+
const presentation = await collectPresentationSnapshot(runtimeFrame, {
|
|
25786
|
+
frameTimes: live.renderStats.frameTimeSamplesMs,
|
|
25787
|
+
hierarchyObjectCount: live.hierarchyObjectCount,
|
|
25788
|
+
renderStats: live.renderStats,
|
|
25789
|
+
warmupFrames: 2,
|
|
25790
|
+
});
|
|
25791
|
+
return {
|
|
25792
|
+
camera: presentation.camera,
|
|
25793
|
+
capture,
|
|
25794
|
+
collectedAt: Date.now(),
|
|
25795
|
+
editor,
|
|
25796
|
+
environment: {
|
|
25797
|
+
...environmentRead.environment,
|
|
25798
|
+
canvas: environmentRead.canvas,
|
|
25799
|
+
},
|
|
25800
|
+
framing: presentation.framing,
|
|
25801
|
+
hierarchy: live.hierarchy,
|
|
25802
|
+
hierarchyObjectCount: presentation.hierarchyObjectCount,
|
|
25803
|
+
nodes: live.nodes,
|
|
25804
|
+
performance: presentation.performance,
|
|
25805
|
+
renderStats: live.renderStats,
|
|
25806
|
+
reloadStartedAt,
|
|
25807
|
+
runtimeReloadStartedAt,
|
|
25808
|
+
runtimeHashes: live.runtimeHashes,
|
|
25809
|
+
};
|
|
25810
|
+
}
|
|
25811
|
+
finally {
|
|
25812
|
+
await renderHold.release();
|
|
25813
|
+
}
|
|
25329
25814
|
});
|
|
25330
25815
|
}
|
|
25331
25816
|
async function inspectPngCapture(bytes) {
|
|
@@ -25407,6 +25892,7 @@ async function launchManagedBrowser(url, headless, verbose, viewport = null, scr
|
|
|
25407
25892
|
managedWorkspaceToken: bootstrap.token,
|
|
25408
25893
|
page,
|
|
25409
25894
|
readiness,
|
|
25895
|
+
renderMode: resolveManagedRenderMode(headless),
|
|
25410
25896
|
waitForRuntime: managedAccess?.runtimeIdentity == null,
|
|
25411
25897
|
signal,
|
|
25412
25898
|
traceMcp,
|
|
@@ -25456,6 +25942,7 @@ async function launchManagedBrowser(url, headless, verbose, viewport = null, scr
|
|
|
25456
25942
|
});
|
|
25457
25943
|
return {
|
|
25458
25944
|
captureRuntimeScreenshot: (options = {}) => commandCoordinator.runExclusive(() => interactionSurface.captureRuntimeScreenshot(options)),
|
|
25945
|
+
captureRuntimeFilmstrip: (options = {}) => commandCoordinator.runExclusive(() => interactionSurface.captureRuntimeFilmstrip(options)),
|
|
25459
25946
|
snapshotApplication: (request = {}) => commandCoordinator.runExclusive(() => interactionSurface.snapshotApplication(request)),
|
|
25460
25947
|
interactApplication: (request) => commandCoordinator.runExclusive(() => interactionSurface.interactApplication(request)),
|
|
25461
25948
|
profileApplication: (request) => commandCoordinator.runExclusive(async () => {
|
|
@@ -60029,6 +60516,7 @@ function iwsdkDev(options = {}) {
|
|
|
60029
60516
|
connectedClientCount: managed.length,
|
|
60030
60517
|
lastTransitionAt: lifecycle.stateEnteredAt,
|
|
60031
60518
|
audioMuted: managedBrowserAudioMuted,
|
|
60519
|
+
renderMode: resolveManagedRenderMode(pluginOptions.workspace?.headless ?? false),
|
|
60032
60520
|
...(lifecycle.issue
|
|
60033
60521
|
? {
|
|
60034
60522
|
lastError: createBrowserIssue(lifecycle.issueCause ??
|
|
@@ -60104,7 +60592,7 @@ function iwsdkDev(options = {}) {
|
|
|
60104
60592
|
currentBrowserState = createBrowserState();
|
|
60105
60593
|
const launchBrowser = () => controller.ensure();
|
|
60106
60594
|
const ensureBrowser = () => controller.ensure();
|
|
60107
|
-
const waitForBridgeConnection = async (timeoutMs, target) => {
|
|
60595
|
+
const waitForBridgeConnection = async (timeoutMs, target, minTabGeneration) => {
|
|
60108
60596
|
const deadline = Date.now() + Math.min(timeoutMs, 5000);
|
|
60109
60597
|
while (!serverShuttingDown && Date.now() < deadline) {
|
|
60110
60598
|
if (listBrowserClients().some((client) => client.deviceClass === 'managed' &&
|
|
@@ -60113,11 +60601,14 @@ function iwsdkDev(options = {}) {
|
|
|
60113
60601
|
client.sessionId === target.sessionId) &&
|
|
60114
60602
|
(target.browserEpoch == null ||
|
|
60115
60603
|
client.browserEpoch === target.browserEpoch) &&
|
|
60604
|
+
(minTabGeneration == null ||
|
|
60605
|
+
client.tabGeneration >= minTabGeneration) &&
|
|
60116
60606
|
client.commandReady)) {
|
|
60117
|
-
return;
|
|
60607
|
+
return true;
|
|
60118
60608
|
}
|
|
60119
60609
|
await controller.wait(controller.snapshot().revision, Math.min(250, deadline - Date.now()));
|
|
60120
60610
|
}
|
|
60611
|
+
return false;
|
|
60121
60612
|
};
|
|
60122
60613
|
// Initialize WebSocket server and client tracking
|
|
60123
60614
|
mcpClients = new Set();
|
|
@@ -60322,12 +60813,28 @@ function iwsdkDev(options = {}) {
|
|
|
60322
60813
|
}, `${options.resultContext}_relaunched`);
|
|
60323
60814
|
return;
|
|
60324
60815
|
}
|
|
60816
|
+
const appBridgeTarget = {
|
|
60817
|
+
role: 'app',
|
|
60818
|
+
sessionId,
|
|
60819
|
+
};
|
|
60820
|
+
const hadAppBridge = options.awaitAppBridge === true &&
|
|
60821
|
+
listBrowserClients().some((client) => client.deviceClass === 'managed' &&
|
|
60822
|
+
client.role === appBridgeTarget.role &&
|
|
60823
|
+
client.commandReady);
|
|
60325
60824
|
const execution = await runManagedBrowserCommand(options.command, readiness.browser, parsed.target, () => options.execute(readiness.browser));
|
|
60825
|
+
// Waited for outside the command lock, whose budget the command
|
|
60826
|
+
// itself may already have used.
|
|
60827
|
+
const appCommandReady = hadAppBridge
|
|
60828
|
+
? await waitForBridgeConnection(BRIDGE_READY_TIMEOUT_MS, {
|
|
60829
|
+
...appBridgeTarget,
|
|
60830
|
+
browserEpoch: controller.snapshot().browserEpoch,
|
|
60831
|
+
}, execution.tab.generation ?? undefined)
|
|
60832
|
+
: undefined;
|
|
60326
60833
|
sendWsJson(ws, {
|
|
60327
60834
|
id: parsed.id,
|
|
60328
60835
|
result: options.serialize == null
|
|
60329
60836
|
? execution.result
|
|
60330
|
-
: options.serialize(execution.result),
|
|
60837
|
+
: options.serialize(execution.result, appCommandReady),
|
|
60331
60838
|
...(execution.tab.id == null
|
|
60332
60839
|
? {}
|
|
60333
60840
|
: {
|
|
@@ -61010,11 +61517,15 @@ function iwsdkDev(options = {}) {
|
|
|
61010
61517
|
errorContext: 'reload_error',
|
|
61011
61518
|
execute: (browser) => browser.reloadApplication(),
|
|
61012
61519
|
resultContext: 'reload_result',
|
|
61013
|
-
serialize: (result) => ({
|
|
61520
|
+
serialize: (result, appCommandReady) => ({
|
|
61014
61521
|
reloaded: true,
|
|
61015
61522
|
url: result.url,
|
|
61523
|
+
...(appCommandReady == null
|
|
61524
|
+
? {}
|
|
61525
|
+
: { commandReady: appCommandReady }),
|
|
61016
61526
|
}),
|
|
61017
61527
|
unavailableContext: 'reload_unavailable',
|
|
61528
|
+
awaitAppBridge: true,
|
|
61018
61529
|
});
|
|
61019
61530
|
return;
|
|
61020
61531
|
}
|
|
@@ -61101,6 +61612,22 @@ function iwsdkDev(options = {}) {
|
|
|
61101
61612
|
});
|
|
61102
61613
|
return;
|
|
61103
61614
|
}
|
|
61615
|
+
if (parsed.method === 'browser_filmstrip' &&
|
|
61616
|
+
typeof parsed.id === 'string') {
|
|
61617
|
+
intercepted = true;
|
|
61618
|
+
await handleManagedBrowserRequest(ws, parsed, {
|
|
61619
|
+
command: 'browser_filmstrip',
|
|
61620
|
+
errorContext: 'browser_filmstrip_error',
|
|
61621
|
+
execute: (browser) => browser.captureRuntimeFilmstrip(parsed.params ?? {}),
|
|
61622
|
+
resultContext: 'browser_filmstrip_result',
|
|
61623
|
+
serialize: (capture) => ({
|
|
61624
|
+
...capture.metadata,
|
|
61625
|
+
imageData: capture.bytes.toString('base64'),
|
|
61626
|
+
}),
|
|
61627
|
+
unavailableContext: 'browser_filmstrip_unavailable',
|
|
61628
|
+
});
|
|
61629
|
+
return;
|
|
61630
|
+
}
|
|
61104
61631
|
if (intercepted &&
|
|
61105
61632
|
typeof parsed.id === 'string' &&
|
|
61106
61633
|
typeof parsed.method === 'string') {
|