getobsrv 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -2
- package/out/cli/args.js +5 -1
- package/out/main/cli.js +64 -9
- package/out/main/index.js +223 -13
- package/out/main/{targetSource-DkXWE0ha.js → targetSource-CbJwfugi.js} +22 -6
- package/out/mcp/server.js +202 -44
- package/out/preload/app.js +10 -2
- package/out/preload/sync.js +102 -5
- package/out/renderer/assets/{index-DyCD4ih_.js → index-BP1S2N6S.js} +81 -4
- package/out/renderer/assets/{index-CHF-G97L.css → index-BeroS4wv.css} +46 -0
- package/out/renderer/index.html +2 -2
- package/out/shared/ipcPayloads.js +61 -4
- package/out/shared/presets.js +7 -1
- package/out/shared/types.js +3 -0
- package/package.json +1 -1
- package/skills/obsrv-screens/SKILL.md +10 -0
package/out/mcp/server.js
CHANGED
|
@@ -11,6 +11,8 @@ const zod_1 = require("zod");
|
|
|
11
11
|
const args_1 = require("../cli/args");
|
|
12
12
|
const control_1 = require("../shared/control");
|
|
13
13
|
const presets_1 = require("../shared/presets");
|
|
14
|
+
const types_1 = require("../shared/types");
|
|
15
|
+
const url_1 = require("../shared/url");
|
|
14
16
|
const control_2 = require("./control");
|
|
15
17
|
const lib_1 = require("./lib");
|
|
16
18
|
/**
|
|
@@ -141,7 +143,16 @@ const snapOutputShape = {
|
|
|
141
143
|
profile: zod_1.z.string().optional().describe('Headless only: applied panel profile id.'),
|
|
142
144
|
settled: zod_1.z
|
|
143
145
|
.boolean()
|
|
144
|
-
.describe('Headless: the page went paint-quiet.
|
|
146
|
+
.describe('Headless: the page went paint-quiet and every pixel painted. False is still a usable capture — a page that ' +
|
|
147
|
+
'kept animating, or one whose repaint never completed, is returned as-is with a warning saying what was ' +
|
|
148
|
+
'missing. Live: the app confirmed the navigation before the capture (trivially true when the app was ' +
|
|
149
|
+
'already showing the URL and nothing was navigated).'),
|
|
150
|
+
navigated: zod_1.z
|
|
151
|
+
.boolean()
|
|
152
|
+
.optional()
|
|
153
|
+
.describe('Live only: false when the app was already showing this URL, so no reload was issued and the capture kept ' +
|
|
154
|
+
'the current scroll position, pan and in-page state. True when the app was pointed somewhere new — that is ' +
|
|
155
|
+
'a fresh load, which starts at the top of the page.'),
|
|
145
156
|
warnings: zod_1.z.array(zod_1.z.string()),
|
|
146
157
|
pngPath: zod_1.z.string().describe('Absolute path of the captured PNG (kept in a per-call temp dir).'),
|
|
147
158
|
url: zod_1.z.string().optional().describe('Live only: the URL the app reports showing.'),
|
|
@@ -179,6 +190,12 @@ const diffInputShape = {
|
|
|
179
190
|
.describe(`Per-render budget for load + paint quiescence, in ms. Default ${args_1.DEFAULT_TIMEOUT_MS}.`),
|
|
180
191
|
};
|
|
181
192
|
const diffOutputShape = {
|
|
193
|
+
settled: zod_1.z
|
|
194
|
+
.boolean()
|
|
195
|
+
.describe('False when either render was a best-effort capture of a page that never stopped painting. The two captures ' +
|
|
196
|
+
'are then different frames, so the band deltas are frame-to-frame noise rather than rendering evidence — ' +
|
|
197
|
+
'`findings` says so instead of interpreting them.'),
|
|
198
|
+
warnings: zod_1.z.array(zod_1.z.string()).describe('Anything either render warned about, prefixed target: / reference:.'),
|
|
182
199
|
url: zod_1.z.string(),
|
|
183
200
|
preset: zod_1.z.string(),
|
|
184
201
|
profile: zod_1.z.string(),
|
|
@@ -233,9 +250,24 @@ const driveInputShape = {
|
|
|
233
250
|
back: zod_1.z.boolean().optional().describe('true: history back (native pane history; the target mirrors the committed page).'),
|
|
234
251
|
forward: zod_1.z.boolean().optional().describe('true: history forward (native pane history; the target mirrors it).'),
|
|
235
252
|
scroll: zod_1.z
|
|
236
|
-
.object({
|
|
253
|
+
.object({
|
|
254
|
+
x: zod_1.z.number().min(0),
|
|
255
|
+
y: zod_1.z.number().min(0),
|
|
256
|
+
scrollSelector: zod_1.z
|
|
257
|
+
.string()
|
|
258
|
+
.min(1)
|
|
259
|
+
.max(types_1.MAX_SCROLL_SELECTOR)
|
|
260
|
+
.optional()
|
|
261
|
+
.describe('Escape hatch: a CSS selector naming the element to scroll, for pages whose scroll host the automatic ' +
|
|
262
|
+
'detection misjudges (several large scrollers, a virtualised list that translates content). No fallback ' +
|
|
263
|
+
'if it matches nothing — the result says so. Same reach as the detection: light DOM of the top-level ' +
|
|
264
|
+
'document only, so a scroller inside a shadow root or an iframe cannot be targeted.'),
|
|
265
|
+
})
|
|
237
266
|
.optional()
|
|
238
|
-
.describe('Scroll both panes to this absolute page offset in CSS px.'
|
|
267
|
+
.describe('Scroll both panes to this absolute page offset in CSS px. Pages whose root cannot scroll (app shells with ' +
|
|
268
|
+
'`html, body { overflow: hidden }` and an inner `overflow-y: auto` container) are handled: the largest ' +
|
|
269
|
+
'visible inner scroller is found and scrolled instead. Check `scrolled` in the result for the offset ' +
|
|
270
|
+
'actually reached — that is how you tell a real scroll from one that clamped.'),
|
|
239
271
|
panTo: zod_1.z
|
|
240
272
|
.object({ x: zod_1.z.number().min(0), y: zod_1.z.number().min(0) })
|
|
241
273
|
.optional()
|
|
@@ -255,6 +287,13 @@ const driveInputShape = {
|
|
|
255
287
|
.optional()
|
|
256
288
|
.describe('Draw a temporary neutral marker over this target-pixel rect in the pane (durationMs default 2000, clamped ' +
|
|
257
289
|
'250-10000). A new highlight replaces the previous one.'),
|
|
290
|
+
capture: zod_1.z
|
|
291
|
+
.enum(['window', 'pane'])
|
|
292
|
+
.optional()
|
|
293
|
+
.describe("Capture the app after the commands run: 'pane' crops to the target pane (the 1x render on its own), " +
|
|
294
|
+
"'window' takes the whole app window. This is how you see a scrolled or panned state — unlike obsrv_snap, " +
|
|
295
|
+
'nothing is navigated, so the scroll position survives. The PNG comes back inline when it is within the ' +
|
|
296
|
+
'1.5 MiB cap, and always as pngPath.'),
|
|
258
297
|
};
|
|
259
298
|
const driveOutputShape = {
|
|
260
299
|
version: zod_1.z.string().describe('The running app version.'),
|
|
@@ -263,6 +302,27 @@ const driveOutputShape = {
|
|
|
263
302
|
profileId: zod_1.z.string(),
|
|
264
303
|
viewMode: zod_1.z.string(),
|
|
265
304
|
mode: zod_1.z.string().describe("The app's pane mode: 'url' (live page) or 'image' (a dropped design export)."),
|
|
305
|
+
scrolled: zod_1.z
|
|
306
|
+
.object({ x: zod_1.z.number(), y: zod_1.z.number() })
|
|
307
|
+
.nullable()
|
|
308
|
+
.optional()
|
|
309
|
+
.describe('Only when `scroll` was requested: the offset the target pane actually reached, read back after the write. ' +
|
|
310
|
+
'Less than you asked for means the content clamped (short page, or the wrong scroller). Null means the ' +
|
|
311
|
+
'pane did not confirm in time — the scroll may still have landed.'),
|
|
312
|
+
scroller: zod_1.z
|
|
313
|
+
.enum(['root', 'element'])
|
|
314
|
+
.optional()
|
|
315
|
+
.describe("Only when `scroll` was requested: 'root' if the document scrolled, 'element' if an inner scroll container did."),
|
|
316
|
+
warnings: zod_1.z.array(zod_1.z.string()).optional().describe('Anything worth knowing about the commands that ran (e.g. a scrollSelector that matched nothing).'),
|
|
317
|
+
pngPath: zod_1.z.string().optional().describe('Only when `capture` was requested: absolute path of the PNG (kept in a per-call temp dir).'),
|
|
318
|
+
width: zod_1.z
|
|
319
|
+
.number()
|
|
320
|
+
.optional()
|
|
321
|
+
.describe('Only when `capture` was requested: captured width in device-independent px; the raster is this times the display scale.'),
|
|
322
|
+
height: zod_1.z
|
|
323
|
+
.number()
|
|
324
|
+
.optional()
|
|
325
|
+
.describe('Only when `capture` was requested: captured height in device-independent px.'),
|
|
266
326
|
};
|
|
267
327
|
// --- live drive --------------------------------------------------------------
|
|
268
328
|
/** Budget for one control `status` round-trip once the app is known live. */
|
|
@@ -287,20 +347,83 @@ function liveFailure(e) {
|
|
|
287
347
|
}
|
|
288
348
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
289
349
|
/**
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
350
|
+
* One short grace before a live capture: the renderer repaints the pane a
|
|
351
|
+
* frame or two after the store confirms, and a capture racing that would show
|
|
352
|
+
* a half-applied flip.
|
|
353
|
+
*/
|
|
354
|
+
const LIVE_CAPTURE_GRACE_MS = 300;
|
|
355
|
+
/**
|
|
356
|
+
* Is the app already showing this page? Compared as parsed URLs so a request
|
|
357
|
+
* for `http://host:5173` matches the `http://host:5173/` the browser commits,
|
|
358
|
+
* and through the same normaliser the URL bar uses so a bare host works too.
|
|
359
|
+
* Anything unparseable falls back to a trimmed string compare.
|
|
360
|
+
*/
|
|
361
|
+
function sameUrl(a, b) {
|
|
362
|
+
const norm = (raw) => {
|
|
363
|
+
const t = raw.trim();
|
|
364
|
+
if (t === '')
|
|
365
|
+
return '';
|
|
366
|
+
try {
|
|
367
|
+
return new URL((0, url_1.normalizeUrl)(t)).href;
|
|
368
|
+
}
|
|
369
|
+
catch {
|
|
370
|
+
return t;
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
const x = norm(a);
|
|
374
|
+
const y = norm(b);
|
|
375
|
+
return x !== '' && x === y;
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Capture the app window — or just the target pane — over the control server
|
|
379
|
+
* and write it to a per-call temp PNG. Shared by the live `obsrv_snap` path
|
|
380
|
+
* and `obsrv_drive`'s `capture`, so both produce byte-identical results.
|
|
381
|
+
*/
|
|
382
|
+
async function liveCapture(info, what) {
|
|
383
|
+
// `pane` crops to the target pane; both answer with the same
|
|
384
|
+
// { data, width, height } shape plus their own warnings (e.g. the pre-mount
|
|
385
|
+
// full-window fallback), which join the tool's.
|
|
386
|
+
const command = what === 'pane' ? 'captureTarget' : 'captureVisible';
|
|
387
|
+
const capture = await (0, control_2.controlCall)(info, command, {}, LIVE_CAPTURE_TIMEOUT_MS);
|
|
388
|
+
const { data, width, height } = capture;
|
|
389
|
+
if (typeof data !== 'string' || typeof width !== 'number' || typeof height !== 'number') {
|
|
390
|
+
throw new Error('the control server returned a malformed capture');
|
|
391
|
+
}
|
|
392
|
+
const warnings = [];
|
|
393
|
+
if (Array.isArray(capture['warnings'])) {
|
|
394
|
+
for (const w of capture['warnings'])
|
|
395
|
+
if (typeof w === 'string')
|
|
396
|
+
warnings.push(w);
|
|
397
|
+
}
|
|
398
|
+
const dir = await (0, promises_1.mkdtemp)((0, node_path_1.join)((0, node_os_1.tmpdir)(), 'obsrv-mcp-'));
|
|
399
|
+
const pngPath = (0, node_path_1.join)(dir, 'live.png');
|
|
400
|
+
await (0, promises_1.writeFile)(pngPath, Buffer.from(data, 'base64'));
|
|
401
|
+
return { pngPath, width, height, warnings };
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* The live `obsrv_snap` path: point the visible app at the URL (plus
|
|
405
|
+
* preset/profile when given), wait — bounded — for it to report the
|
|
406
|
+
* navigation, then capture the window exactly as the user sees it.
|
|
407
|
+
*
|
|
408
|
+
* When the app is already showing that URL the navigation is skipped entirely.
|
|
409
|
+
* A navigate is a fresh `loadURL`, which resets the scroll position, so
|
|
410
|
+
* reloading here would make `obsrv_drive { scroll }` followed by a snap of the
|
|
411
|
+
* same page always capture the top. `navigated: false` says which happened.
|
|
293
412
|
*/
|
|
294
413
|
async function liveSnap(app, input, notes) {
|
|
295
414
|
const { info } = app;
|
|
296
415
|
const warnings = [...notes];
|
|
297
416
|
const before = app.status.url;
|
|
298
|
-
|
|
417
|
+
// Already there? Then leave the page alone — see the note above.
|
|
418
|
+
const navigated = !sameUrl(before, input.url);
|
|
419
|
+
let applied = before;
|
|
299
420
|
try {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
421
|
+
if (navigated) {
|
|
422
|
+
// The navigate command answers once both panes finished loading, so it
|
|
423
|
+
// carries the same per-render budget the headless path polices.
|
|
424
|
+
const nav = await (0, control_2.controlCall)(info, 'navigate', { url: input.url.trim() }, (input.timeoutMs ?? args_1.DEFAULT_TIMEOUT_MS) + 10_000);
|
|
425
|
+
applied = typeof nav['url'] === 'string' ? nav['url'] : '';
|
|
426
|
+
}
|
|
304
427
|
if (input.preset !== undefined)
|
|
305
428
|
await (0, control_2.controlCall)(info, 'setPreset', { id: input.preset }, LIVE_APPLY_TIMEOUT_MS);
|
|
306
429
|
if (input.profile !== undefined)
|
|
@@ -311,15 +434,18 @@ async function liveSnap(app, input, notes) {
|
|
|
311
434
|
}
|
|
312
435
|
// The app settles when it reports the applied URL — or, after a redirect,
|
|
313
436
|
// any committed non-blank URL that is no longer the pre-navigation one.
|
|
437
|
+
// Nothing to settle when no navigation was issued; one status read still
|
|
438
|
+
// refreshes the preset/profile/view the result reports.
|
|
314
439
|
let status = app.status;
|
|
315
|
-
let settled =
|
|
440
|
+
let settled = !navigated;
|
|
316
441
|
const deadline = Date.now() + LIVE_SETTLE_MS;
|
|
317
442
|
for (;;) {
|
|
318
443
|
try {
|
|
319
444
|
const s = (0, control_1.parseControlStatus)(await (0, control_2.controlCall)(info, 'status', {}, LIVE_STATUS_TIMEOUT_MS));
|
|
320
445
|
if (s) {
|
|
321
446
|
status = s;
|
|
322
|
-
|
|
447
|
+
if (navigated)
|
|
448
|
+
settled = s.url === applied || (applied !== '' && s.url !== before && s.url !== 'about:blank');
|
|
323
449
|
}
|
|
324
450
|
}
|
|
325
451
|
catch (e) {
|
|
@@ -331,33 +457,16 @@ async function liveSnap(app, input, notes) {
|
|
|
331
457
|
}
|
|
332
458
|
if (!settled)
|
|
333
459
|
warnings.push('the app did not confirm the navigation before capture; the PNG may show the previous page.');
|
|
334
|
-
|
|
335
|
-
// (and any preset resize) a frame or two after the store confirms, and a
|
|
336
|
-
// capture racing that would show a half-applied flip.
|
|
337
|
-
await sleep(300);
|
|
338
|
-
// `capture: 'pane'` crops to the target pane; the command answers with the
|
|
339
|
-
// same { data, width, height } shape plus its own warnings (e.g. the
|
|
340
|
-
// pre-mount full-window fallback), which join the tool's.
|
|
460
|
+
await sleep(LIVE_CAPTURE_GRACE_MS);
|
|
341
461
|
let capture;
|
|
342
462
|
try {
|
|
343
|
-
|
|
344
|
-
capture = await (0, control_2.controlCall)(info, command, {}, LIVE_CAPTURE_TIMEOUT_MS);
|
|
463
|
+
capture = await liveCapture(info, input.capture === 'pane' ? 'pane' : 'window');
|
|
345
464
|
}
|
|
346
465
|
catch (e) {
|
|
347
466
|
return toolError(liveFailure(e));
|
|
348
467
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
return toolError('the control server returned a malformed capture');
|
|
352
|
-
}
|
|
353
|
-
if (Array.isArray(capture['warnings'])) {
|
|
354
|
-
for (const w of capture['warnings'])
|
|
355
|
-
if (typeof w === 'string')
|
|
356
|
-
warnings.push(w);
|
|
357
|
-
}
|
|
358
|
-
const dir = await (0, promises_1.mkdtemp)((0, node_path_1.join)((0, node_os_1.tmpdir)(), 'obsrv-mcp-'));
|
|
359
|
-
const pngPath = (0, node_path_1.join)(dir, 'live.png');
|
|
360
|
-
await (0, promises_1.writeFile)(pngPath, Buffer.from(data, 'base64'));
|
|
468
|
+
warnings.push(...capture.warnings);
|
|
469
|
+
const { pngPath, width, height } = capture;
|
|
361
470
|
const structured = {
|
|
362
471
|
mode: 'live',
|
|
363
472
|
url: status.url,
|
|
@@ -367,6 +476,7 @@ async function liveSnap(app, input, notes) {
|
|
|
367
476
|
width,
|
|
368
477
|
height,
|
|
369
478
|
settled,
|
|
479
|
+
navigated,
|
|
370
480
|
warnings,
|
|
371
481
|
pngPath,
|
|
372
482
|
};
|
|
@@ -398,7 +508,10 @@ server.registerTool('obsrv_snap', {
|
|
|
398
508
|
`ignored in live mode. \`mode: "live"\` errors when the app is not reachable; \`mode: "headless"\` never ` +
|
|
399
509
|
`touches it. Note: although this tool is annotated read-only (it renders and captures), a live snap steers ` +
|
|
400
510
|
`the open app window — navigating it and flipping its preset in front of the user — as its means of ` +
|
|
401
|
-
`capture; that visible steering is the point of live mode
|
|
511
|
+
`capture; that visible steering is the point of live mode.\n\n` +
|
|
512
|
+
`A live snap only navigates when the app is showing a different URL; the result's \`navigated\` says which ` +
|
|
513
|
+
`happened. Navigating is a fresh load, so it starts at the top of the page — to photograph a scrolled or ` +
|
|
514
|
+
`panned state, use obsrv_drive with \`capture\` instead, which never navigates unless you ask it to.`,
|
|
402
515
|
inputSchema: snapInputShape,
|
|
403
516
|
outputSchema: snapOutputShape,
|
|
404
517
|
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
|
|
@@ -462,6 +575,9 @@ server.registerTool('obsrv_diff', {
|
|
|
462
575
|
`8 horizontal band deltas with humanised findings (informational — apply your own thresholds), and the ` +
|
|
463
576
|
`paths of target.png / reference.png in a per-call temp dir. \`includeImages: true\` also inlines both ` +
|
|
464
577
|
`PNGs (1.5 MiB cap each).\n\n` +
|
|
578
|
+
`Check \`settled\` before believing the bands: a page that never stops painting (animation, video) yields ` +
|
|
579
|
+
`two captures of *different frames*, so every delta is frame-to-frame noise. When it is false the numbers ` +
|
|
580
|
+
`are still returned but \`findings\` says so instead of interpreting them.\n\n` +
|
|
465
581
|
`1x presets only (e.g. laptop-768, 1080p-24): dense presets (phones) and CSS viewports over 2048px are ` +
|
|
466
582
|
`refused with an explanatory error — use obsrv_snap for those.\n\n` +
|
|
467
583
|
`Headless-only: a diff always performs its own two renders and never drives a running Obsrv app window ` +
|
|
@@ -502,15 +618,22 @@ server.registerTool('obsrv_drive', {
|
|
|
502
618
|
`both panes, pan the target pane to a pixel, click the live page, and highlight a rect with a temporary ` +
|
|
503
619
|
`neutral marker, all while the user watches.\n\n` +
|
|
504
620
|
`Only the supplied inputs run (none = just read the current state), in this fixed order: focus → url → ` +
|
|
505
|
-
`preset → profile → viewMode → pixelExact → reload → back → forward → scroll → panTo → click → highlight
|
|
621
|
+
`preset → profile → viewMode → pixelExact → reload → back → forward → scroll → panTo → click → highlight → ` +
|
|
622
|
+
`capture. ` +
|
|
506
623
|
`The result is the final status: app version, the URL showing, and the selected preset/profile/view. A ` +
|
|
507
|
-
`click that navigates is reflected in that status — the call waits briefly (up to 2 s) for the commit. ` +
|
|
624
|
+
`click that navigates is reflected in that status — the call waits briefly (up to 2 s) for the commit. A ` +
|
|
625
|
+
`scroll adds \`scrolled\` (the offset actually reached) and \`scroller\` ('root' or 'element'): compare ` +
|
|
626
|
+
`\`scrolled\` with what you asked for rather than trusting the call's success, and use \`scroll.scrollSelector\` ` +
|
|
627
|
+
`when the automatic scroll-host detection picks the wrong container.\n\n` +
|
|
508
628
|
`Coordinates: click takes CSS-viewport px of the page (the valid range is 0 up to but not including the ` +
|
|
509
629
|
`viewport size); panTo and highlight take target-pane pixels (device px of the render — identical to CSS px ` +
|
|
510
630
|
`on 1x presets); scroll takes page CSS px.\n\n` +
|
|
631
|
+
`Pass \`capture\` to get a PNG back once the commands have run. Nothing in this tool navigates unless you ` +
|
|
632
|
+
`pass \`url\`, so this is how you photograph a scrolled or panned state: scroll, then capture, in one call. ` +
|
|
633
|
+
`obsrv_snap is the other way round — it points the app at a URL first, and pointing it somewhere new is a ` +
|
|
634
|
+
`fresh load that starts at the top.\n\n` +
|
|
511
635
|
`Requires the app to be open with its "Agent control" toolbar toggle on; errors otherwise. This tool ` +
|
|
512
|
-
`mutates visible app state (it changes what the user's window shows, and a click can act on the live page)
|
|
513
|
-
`but renders nothing itself — use obsrv_snap for a capture.`,
|
|
636
|
+
`mutates visible app state (it changes what the user's window shows, and a click can act on the live page).`,
|
|
514
637
|
inputSchema: driveInputShape,
|
|
515
638
|
outputSchema: driveOutputShape,
|
|
516
639
|
// Honest annotation: this changes what the user's window is showing.
|
|
@@ -548,8 +671,25 @@ server.registerTool('obsrv_drive', {
|
|
|
548
671
|
await (0, control_2.controlCall)(live.info, 'back', {}, LIVE_APPLY_TIMEOUT_MS);
|
|
549
672
|
if (input.forward)
|
|
550
673
|
await (0, control_2.controlCall)(live.info, 'forward', {}, LIVE_APPLY_TIMEOUT_MS);
|
|
551
|
-
|
|
552
|
-
|
|
674
|
+
// The scroll answer is the interesting half: it reports the offset the
|
|
675
|
+
// pane reached, which is the only way to tell a scroll from a clamp.
|
|
676
|
+
let scrolled;
|
|
677
|
+
let scroller;
|
|
678
|
+
const warnings = [];
|
|
679
|
+
if (input.scroll !== undefined) {
|
|
680
|
+
const r = await (0, control_2.controlCall)(live.info, 'scroll', input.scroll, LIVE_APPLY_TIMEOUT_MS);
|
|
681
|
+
const at = r['scrolled'];
|
|
682
|
+
scrolled =
|
|
683
|
+
at !== null && typeof at === 'object' && typeof at.x === 'number' && typeof at.y === 'number'
|
|
684
|
+
? { x: at.x, y: at.y }
|
|
685
|
+
: null;
|
|
686
|
+
if (r['scroller'] === 'root' || r['scroller'] === 'element')
|
|
687
|
+
scroller = r['scroller'];
|
|
688
|
+
if (Array.isArray(r['warnings']))
|
|
689
|
+
for (const w of r['warnings'])
|
|
690
|
+
if (typeof w === 'string')
|
|
691
|
+
warnings.push(w);
|
|
692
|
+
}
|
|
553
693
|
if (input.panTo !== undefined)
|
|
554
694
|
await (0, control_2.controlCall)(live.info, 'panTo', input.panTo, LIVE_APPLY_TIMEOUT_MS);
|
|
555
695
|
if (input.click !== undefined) {
|
|
@@ -571,13 +711,31 @@ server.registerTool('obsrv_drive', {
|
|
|
571
711
|
}
|
|
572
712
|
if (input.highlight !== undefined)
|
|
573
713
|
await (0, control_2.controlCall)(live.info, 'highlight', input.highlight, LIVE_APPLY_TIMEOUT_MS);
|
|
714
|
+
// Capture last, so the PNG shows everything the commands above did.
|
|
715
|
+
// Nothing here navigates, so a scroll or pan applied in this same call
|
|
716
|
+
// is still in place when the shutter fires.
|
|
717
|
+
let capture = null;
|
|
718
|
+
if (input.capture !== undefined) {
|
|
719
|
+
await sleep(LIVE_CAPTURE_GRACE_MS);
|
|
720
|
+
capture = await liveCapture(live.info, input.capture);
|
|
721
|
+
warnings.push(...capture.warnings);
|
|
722
|
+
}
|
|
574
723
|
const status = (0, control_1.parseControlStatus)(await (0, control_2.controlCall)(live.info, 'status', {}, LIVE_STATUS_TIMEOUT_MS));
|
|
575
724
|
if (!status)
|
|
576
725
|
return toolError('the control server returned a malformed status');
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
726
|
+
const structured = {
|
|
727
|
+
...status,
|
|
728
|
+
...(input.scroll !== undefined ? { scrolled: scrolled ?? null } : {}),
|
|
729
|
+
...(scroller !== undefined ? { scroller } : {}),
|
|
730
|
+
...(warnings.length > 0 ? { warnings } : {}),
|
|
731
|
+
...(capture !== null ? { pngPath: capture.pngPath, width: capture.width, height: capture.height } : {}),
|
|
580
732
|
};
|
|
733
|
+
const content = [{ type: 'text', text: JSON.stringify(structured, null, 2) }];
|
|
734
|
+
if (capture !== null) {
|
|
735
|
+
const label = input.capture === 'pane' ? 'The captured target pane' : 'The captured app window';
|
|
736
|
+
content.push(await imageOrNote(capture.pngPath, label, 'read the file at pngPath'));
|
|
737
|
+
}
|
|
738
|
+
return { content, structuredContent: structured };
|
|
581
739
|
}
|
|
582
740
|
catch (e) {
|
|
583
741
|
return toolError(liveFailure(e));
|
package/out/preload/app.js
CHANGED
|
@@ -25,7 +25,11 @@ const IPC = {
|
|
|
25
25
|
readImageFile: "obsrv:read-image-file",
|
|
26
26
|
uiState: "obsrv:ui-state",
|
|
27
27
|
agentApply: "obsrv:agent-apply",
|
|
28
|
-
agentActivity: "obsrv:agent-activity"
|
|
28
|
+
agentActivity: "obsrv:agent-activity",
|
|
29
|
+
getUpdate: "obsrv:get-update",
|
|
30
|
+
checkUpdate: "obsrv:check-update",
|
|
31
|
+
openRelease: "obsrv:open-release",
|
|
32
|
+
updateStatus: "obsrv:update-status"
|
|
29
33
|
};
|
|
30
34
|
function subscribe(channel, cb) {
|
|
31
35
|
const listener = (_e, v) => cb(v);
|
|
@@ -94,6 +98,10 @@ const api = {
|
|
|
94
98
|
return () => {
|
|
95
99
|
electron.ipcRenderer.removeListener(IPC.agentActivity, listener);
|
|
96
100
|
};
|
|
97
|
-
}
|
|
101
|
+
},
|
|
102
|
+
getUpdate: () => electron.ipcRenderer.invoke(IPC.getUpdate),
|
|
103
|
+
checkUpdate: () => electron.ipcRenderer.invoke(IPC.checkUpdate),
|
|
104
|
+
openRelease: () => electron.ipcRenderer.invoke(IPC.openRelease),
|
|
105
|
+
onUpdateStatus: (cb) => subscribe(IPC.updateStatus, cb)
|
|
98
106
|
};
|
|
99
107
|
electron.contextBridge.exposeInMainWorld("obsrv", api);
|
package/out/preload/sync.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
3
|
const electron = require("electron");
|
|
3
4
|
const SUPPRESS_MS = 120;
|
|
4
5
|
const SYNC_SCROLL = "obsrv:sync-scroll";
|
|
5
6
|
const APPLY_SCROLL = "obsrv:apply-scroll";
|
|
7
|
+
const SCROLL_RESULT = "obsrv:scroll-result";
|
|
8
|
+
const MAX_VISITED = 2e3;
|
|
9
|
+
const SCROLL_EPSILON = 1;
|
|
6
10
|
let suppressUntil = 0;
|
|
7
11
|
let lastApplied = null;
|
|
8
12
|
let rafId = 0;
|
|
@@ -33,9 +37,102 @@ window.addEventListener(
|
|
|
33
37
|
},
|
|
34
38
|
{ passive: true }
|
|
35
39
|
);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
function rootScrolls() {
|
|
41
|
+
const el = document.scrollingElement;
|
|
42
|
+
if (!el) return false;
|
|
43
|
+
return el.scrollHeight > el.clientHeight + SCROLL_EPSILON || el.scrollWidth > el.clientWidth + SCROLL_EPSILON;
|
|
44
|
+
}
|
|
45
|
+
function canScroll(el) {
|
|
46
|
+
const overflowsY = el.scrollHeight > el.clientHeight + SCROLL_EPSILON;
|
|
47
|
+
const overflowsX = el.scrollWidth > el.clientWidth + SCROLL_EPSILON;
|
|
48
|
+
if (!overflowsY && !overflowsX) return false;
|
|
49
|
+
const style = window.getComputedStyle(el);
|
|
50
|
+
const scrollableY = style.overflowY === "auto" || style.overflowY === "scroll";
|
|
51
|
+
const scrollableX = style.overflowX === "auto" || style.overflowX === "scroll";
|
|
52
|
+
return overflowsY && scrollableY || overflowsX && scrollableX;
|
|
53
|
+
}
|
|
54
|
+
function isVisible(el) {
|
|
55
|
+
const check = el.checkVisibility;
|
|
56
|
+
if (typeof check !== "function") return el.getClientRects().length > 0;
|
|
57
|
+
return check.call(el, { visibilityProperty: true, opacityProperty: true });
|
|
58
|
+
}
|
|
59
|
+
function findScroller(root = document.body) {
|
|
60
|
+
if (!root) return null;
|
|
61
|
+
let best = null;
|
|
62
|
+
let bestArea = 0;
|
|
63
|
+
let visited = 0;
|
|
64
|
+
const stack = [root];
|
|
65
|
+
while (stack.length > 0) {
|
|
66
|
+
const el = stack.pop();
|
|
67
|
+
if (visited++ >= MAX_VISITED) break;
|
|
68
|
+
const area = el.clientWidth * el.clientHeight;
|
|
69
|
+
if (area <= 0 && el.getClientRects().length === 0 && window.getComputedStyle(el).display === "none") continue;
|
|
70
|
+
if (area > bestArea && canScroll(el) && isVisible(el)) {
|
|
71
|
+
best = el;
|
|
72
|
+
bestArea = area;
|
|
73
|
+
}
|
|
74
|
+
const kids = el.children;
|
|
75
|
+
for (let i = kids.length - 1; i >= 0; i--) stack.push(kids[i]);
|
|
76
|
+
}
|
|
77
|
+
return best;
|
|
78
|
+
}
|
|
79
|
+
let cachedScroller = null;
|
|
80
|
+
function resolveScroller() {
|
|
81
|
+
if (rootScrolls()) {
|
|
82
|
+
cachedScroller = null;
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
if (cachedScroller && cachedScroller.isConnected && canScroll(cachedScroller)) return cachedScroller;
|
|
86
|
+
cachedScroller = findScroller();
|
|
87
|
+
return cachedScroller;
|
|
88
|
+
}
|
|
89
|
+
function applyTo(el, pos) {
|
|
90
|
+
if (el) {
|
|
91
|
+
el.scrollTo({ left: pos.x, top: pos.y, behavior: "instant" });
|
|
92
|
+
return { x: el.scrollLeft, y: el.scrollTop };
|
|
93
|
+
}
|
|
94
|
+
window.scrollTo({ left: pos.x, top: pos.y, behavior: "instant" });
|
|
95
|
+
return { x: window.scrollX, y: window.scrollY };
|
|
96
|
+
}
|
|
97
|
+
electron.ipcRenderer.on(APPLY_SCROLL, (_e, req) => {
|
|
98
|
+
const pos = { x: req.x, y: req.y };
|
|
99
|
+
const warnings = [];
|
|
100
|
+
let scroller = "root";
|
|
101
|
+
let reached;
|
|
102
|
+
if (typeof req.selector === "string") {
|
|
103
|
+
let el = null;
|
|
104
|
+
try {
|
|
105
|
+
el = document.querySelector(req.selector);
|
|
106
|
+
} catch {
|
|
107
|
+
warnings.push(`scrollSelector ${JSON.stringify(req.selector)} is not a valid CSS selector; nothing was scrolled`);
|
|
108
|
+
}
|
|
109
|
+
if (el) {
|
|
110
|
+
scroller = "element";
|
|
111
|
+
reached = applyTo(el, pos);
|
|
112
|
+
if (reached.x !== pos.x || reached.y !== pos.y) {
|
|
113
|
+
warnings.push(
|
|
114
|
+
`scrollSelector ${JSON.stringify(req.selector)} matched an element that could not reach (${pos.x}, ${pos.y}); it stopped at (${reached.x}, ${reached.y})`
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
} else {
|
|
118
|
+
if (warnings.length === 0) {
|
|
119
|
+
warnings.push(`scrollSelector ${JSON.stringify(req.selector)} matched no element; nothing was scrolled`);
|
|
120
|
+
}
|
|
121
|
+
reached = { x: window.scrollX, y: window.scrollY };
|
|
122
|
+
}
|
|
123
|
+
} else {
|
|
124
|
+
const el = resolveScroller();
|
|
125
|
+
scroller = el ? "element" : "root";
|
|
126
|
+
if (!el) {
|
|
127
|
+
lastApplied = pos;
|
|
128
|
+
if (window.scrollX !== pos.x || window.scrollY !== pos.y) suppressUntil = performance.now() + SUPPRESS_MS;
|
|
129
|
+
}
|
|
130
|
+
reached = applyTo(el, pos);
|
|
131
|
+
}
|
|
132
|
+
if (typeof req.id === "number") {
|
|
133
|
+
electron.ipcRenderer.send(SCROLL_RESULT, { id: req.id, x: reached.x, y: reached.y, scroller, warnings });
|
|
134
|
+
}
|
|
41
135
|
});
|
|
136
|
+
exports.MAX_VISITED = MAX_VISITED;
|
|
137
|
+
exports.findScroller = findScroller;
|
|
138
|
+
exports.resolveScroller = resolveScroller;
|