@classic-homes/page-agent-core 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -7
- package/dist/index.d.ts +148 -5
- package/dist/index.js +215 -35
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,7 +43,7 @@ durationMs })`, `point(ref, { label, durationMs })`, `scrollToTarget(ref)`,
|
|
|
43
43
|
`attachPageAgent()` wires Escape and `popstate` to clear; call
|
|
44
44
|
`notifyNavigated()` after SPA navigation.
|
|
45
45
|
- **Executor**: `configurePageAgent({ navigate, aliasAttributes,
|
|
46
|
-
ignoreSelector })` and `executePageCall(call)`, the single entry point for
|
|
46
|
+
ignoreSelector, agentUiSelector, clearTheWay })` and `executePageCall(call)`, the single entry point for
|
|
47
47
|
the model's tool calls. `PAGE_TOOL_NAMES` and `PAGE_TOOLS` (plain-JSON
|
|
48
48
|
input shapes) let the agent server mirror the contract exactly.
|
|
49
49
|
- **Visibility**: `measureVisibility(el)` says what of a target the user can
|
|
@@ -57,10 +57,16 @@ ignoreSelector })` and `executePageCall(call)`, the single entry point for
|
|
|
57
57
|
`inert`. Bindings draw only `visible` and hide (not
|
|
58
58
|
remove) effects while it is null. The maths is the pure
|
|
59
59
|
`computeVisibility({ rect, viewport, clips, behindModal, coverAt })`.
|
|
60
|
+
A target hidden only by the agent's own UI (`agentUiSelector`) is
|
|
61
|
+
`coveredBy: 'agent'`; whenever that UI hides all or part of it, the
|
|
62
|
+
result carries `agentCover` and `visibleWithoutAgent`.
|
|
60
63
|
- **Collapsed tab rows**: `collapsedTabMenu(tab)` and the
|
|
61
64
|
`data-tablist-menu` DOM contract (below).
|
|
62
65
|
- **Geometry**: `unionRects`, `inflate`, `clampToViewport`, `intersectBoxes`,
|
|
63
|
-
`placeLabel` (with optional `bounds`), `pointerTarget` for the UI layer
|
|
66
|
+
`placeLabel` (with optional `bounds`), `pointerTarget` for the UI layer,
|
|
67
|
+
and `findClearPosition(viewport, windowRect, keepClear, { margin, edge })`,
|
|
68
|
+
`overlaps(a, b, margin)` and `CLEAR_MARGIN` for hosts moving their chat
|
|
69
|
+
window or dock off a target (below).
|
|
64
70
|
|
|
65
71
|
```ts
|
|
66
72
|
import {
|
|
@@ -105,8 +111,110 @@ output? }` with `status` one of `ok`, `not_found`, `ambiguous`, `hidden`,
|
|
|
105
111
|
`highlight`, `point` and `scroll_to` wait for their scroll to settle, scroll
|
|
106
112
|
once more if a sticky header or footer still covers the target, and measure
|
|
107
113
|
`inViewport` after that. `highlight` / `point` on a target the user cannot
|
|
108
|
-
see because something covers it return `hidden` with `output.reason`
|
|
109
|
-
`
|
|
114
|
+
see because something covers it return `hidden` with `output.reason` (type
|
|
115
|
+
`CoveredReason`, exactly these three):
|
|
116
|
+
|
|
117
|
+
| `output.reason` | Covered by |
|
|
118
|
+
| ------------------ | ----------------------------------------------------------------------------- |
|
|
119
|
+
| `behind_dialog` | A modal dialog, or the target is in an `inert` subtree |
|
|
120
|
+
| `covered` | Another part of the page (a header, panel, popup) that scrolling cannot clear |
|
|
121
|
+
| `covered_by_agent` | The agent's own UI (`agentUiSelector`): its chat window, dock or launcher |
|
|
122
|
+
|
|
123
|
+
`scroll_to` also returns `hidden` / `covered_by_agent` when its target ends up
|
|
124
|
+
wholly under the agent's UI; under any other cover it still answers `ok` with
|
|
125
|
+
`covered: true`. A target only partly under the agent's UI is `ok`, with
|
|
126
|
+
`output.partlyCoveredByAgent: true`.
|
|
127
|
+
|
|
128
|
+
## The agent's own UI
|
|
129
|
+
|
|
130
|
+
The agent's chat window, or a panel docked to one edge that overlaps the
|
|
131
|
+
page, can sit over the very thing the user asked about. Tell the core which
|
|
132
|
+
elements are that UI; it then sees beneath it, says when it is in the way,
|
|
133
|
+
and lets you move it:
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
import {
|
|
137
|
+
configurePageAgent,
|
|
138
|
+
findClearPosition,
|
|
139
|
+
overlaps,
|
|
140
|
+
viewportSize,
|
|
141
|
+
CLEAR_MARGIN,
|
|
142
|
+
type ClearTheWayHandler,
|
|
143
|
+
} from '@classic-homes/page-agent-core';
|
|
144
|
+
|
|
145
|
+
// Your chat: floatAt(point), setDockEdge('left' | 'right'), and its state.
|
|
146
|
+
const clearTheWay: ClearTheWayHandler = ({ rect, cover }) => {
|
|
147
|
+
const vp = viewportSize();
|
|
148
|
+
if (chat.mode === 'dock') {
|
|
149
|
+
// Would the target be clear with the dock on the other edge?
|
|
150
|
+
const other = chat.edge === 'left' ? 'right' : 'left';
|
|
151
|
+
const there = {
|
|
152
|
+
top: 0,
|
|
153
|
+
height: vp.height,
|
|
154
|
+
width: chat.dockWidth,
|
|
155
|
+
left: other === 'left' ? 0 : vp.width - chat.dockWidth,
|
|
156
|
+
};
|
|
157
|
+
if (!overlaps(there, rect, CLEAR_MARGIN)) {
|
|
158
|
+
chat.setDockEdge(other);
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// Floating (or no free edge: turn the dock into the floating window).
|
|
163
|
+
const size = chat.floatSize; // { width, height } of the floating window
|
|
164
|
+
const from = chat.mode === 'dock' ? { ...cover.rect, ...size } : cover.rect;
|
|
165
|
+
const to = findClearPosition(vp, from, rect); // default margin: CLEAR_MARGIN
|
|
166
|
+
if (!to) return false; // nowhere to go: the effect is cut, or covered_by_agent
|
|
167
|
+
chat.floatAt(to);
|
|
168
|
+
return true;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
configurePageAgent({ agentUiSelector: '[data-assistant-ui]', clearTheWay });
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
- `agentUiSelector` (default none) is never read or pointed at, like
|
|
175
|
+
`ignoreSelector`. It is a separate option because neither
|
|
176
|
+
`ignoreSelector` nor `[data-agent-ignore]` means "the agent's UI":
|
|
177
|
+
`data-agent-ignore` also marks sensitive page regions (an MFA dialog), and
|
|
178
|
+
a target under one of those is plainly `covered`.
|
|
179
|
+
- **Everything else sees beneath it.** `list_targets` / `listTargets()` and
|
|
180
|
+
`listTabs()` report targets under the panel as `visible` and `inViewport`
|
|
181
|
+
(they never hit-test), `readPageText` reads the page text under it (never
|
|
182
|
+
the panel's own), resolution by id or text finds targets under it, and
|
|
183
|
+
`select_tab` clicks a tab under it.
|
|
184
|
+
- **Wholly hidden**: a target counts as `covered_by_agent` only when it
|
|
185
|
+
would be at least partly visible without the agent's UI; under the chat
|
|
186
|
+
window _and_ a sticky header, it is `covered`. `highlight`, `point` and
|
|
187
|
+
`scroll_to` then answer `hidden` with that reason.
|
|
188
|
+
- **Partly hidden**: when the agent's UI covers part of the target (a dock
|
|
189
|
+
over the right 40% of a heading cuts it; a window over a corner is drawn
|
|
190
|
+
over), the call still answers `ok`, with the effect cut at the panel, and
|
|
191
|
+
`output.partlyCoveredByAgent: true` when that is still so at the end.
|
|
192
|
+
- `clearTheWay(request)` is called at most once per `highlight`, `point`,
|
|
193
|
+
`scroll_to` or `select_tab`, whenever the agent's UI hides all or part of
|
|
194
|
+
the target, never for other covers. `request` is `{ tool, target: { id?,
|
|
195
|
+
label }, rect, cover: { element, rect }, viewport }`, all in viewport px:
|
|
196
|
+
`rect` is what the user would see of the target without the agent's UI
|
|
197
|
+
(`visibleWithoutAgent`), `cover.element` the outermost `agentUiSelector`
|
|
198
|
+
element hit and `cover.rect` its box over the target. Return (or resolve
|
|
199
|
+
to) `true` when you moved something; the call then waits for the target
|
|
200
|
+
and that element to stop moving (at most 1 s) and measures once more
|
|
201
|
+
(`select_tab` clicks straight away). No answer within
|
|
202
|
+
`CLEAR_THE_WAY_TIMEOUT_MS` (1000), a throw or a rejection count as "did
|
|
203
|
+
nothing". Results after a move carry `output.clearedTheWay: true`.
|
|
204
|
+
- **Keep room for the caption.** An outline's caption sits up to 34 px above
|
|
205
|
+
(or below) `rect`. `findClearPosition`'s default margin, `CLEAR_MARGIN`
|
|
206
|
+
(40), allows for it; use the same margin with `overlaps`. A ghost
|
|
207
|
+
cursor's label can reach further right (up to 240 px); pass a larger
|
|
208
|
+
margin for `point` if your window sits right of the target.
|
|
209
|
+
- Effects already on screen that later end up under the agent's UI (the
|
|
210
|
+
user drags the window over an outline) are hidden or cut as before; the
|
|
211
|
+
hook is not called for them.
|
|
212
|
+
- `findClearPosition(viewport, windowRect, keepClear, { margin, edge })`
|
|
213
|
+
returns the nearest top-left position for a floating window that keeps
|
|
214
|
+
`margin` px (default 40) from `keepClear` and `edge` px (default 8) from
|
|
215
|
+
the viewport edges (the current one if it already does), or null when
|
|
216
|
+
none fits. `overlaps(a, b, margin)` answers "is `a` in the way of `b`
|
|
217
|
+
plus `margin`?" for a dock at the other edge.
|
|
110
218
|
|
|
111
219
|
## Collapsed tab rows
|
|
112
220
|
|
|
@@ -145,8 +253,8 @@ attribute, and nothing but `role="tab"` is ever clicked.
|
|
|
145
253
|
- `ok` is reported only once the effect is in place (the effect is in the
|
|
146
254
|
store and its target is not covered, the tab is selected, the host's
|
|
147
255
|
navigation resolved).
|
|
148
|
-
- `[data-agent-ignore]`, the host `ignoreSelector` and
|
|
149
|
-
(`.chpa`) are invisible to resolution, snapshots and `readPageText`, which
|
|
256
|
+
- `[data-agent-ignore]`, the host `ignoreSelector` and `agentUiSelector`,
|
|
257
|
+
and the layer's own UI (`.chpa`) are invisible to resolution, snapshots and `readPageText`, which
|
|
150
258
|
also never reads password, hidden, card or one-time-code field values.
|
|
151
259
|
Treat page text as untrusted data on the server.
|
|
152
260
|
|
|
@@ -172,4 +280,6 @@ jsdom has no layout, so pass `configurePageAgent({ measure })` to supply
|
|
|
172
280
|
element boxes. The style rules (`hidden`, `display: none`, `visibility`,
|
|
173
281
|
`opacity: 0`) still run for real. jsdom has no hit testing either: nothing
|
|
174
282
|
counts as covered unless you pass `configurePageAgent({ hitTest: (x, y) =>
|
|
175
|
-
element | null })`.
|
|
283
|
+
element | null })`. A custom `hitTest` cannot look through the agent's UI,
|
|
284
|
+
so points it reports under `agentUiSelector` count as uncovered when
|
|
285
|
+
telling `covered_by_agent` from `covered`.
|
package/dist/index.d.ts
CHANGED
|
@@ -107,6 +107,13 @@ type PageCall = {
|
|
|
107
107
|
};
|
|
108
108
|
type PageToolName = PageCall['tool'];
|
|
109
109
|
type PageCallStatus = 'ok' | 'not_found' | 'ambiguous' | 'hidden' | 'refused' | 'error';
|
|
110
|
+
/**
|
|
111
|
+
* `output.reason` of a `hidden` result for a covered target: behind a modal
|
|
112
|
+
* dialog, under another part of the page, or under the agent's own UI (see
|
|
113
|
+
* {@link PageAgentConfig.agentUiSelector}). A closed set: agent servers
|
|
114
|
+
* mirror it as an enum.
|
|
115
|
+
*/
|
|
116
|
+
type CoveredReason = 'behind_dialog' | 'covered' | 'covered_by_agent';
|
|
110
117
|
/** What goes back to the model. `ok` only once the effect is confirmed. */
|
|
111
118
|
interface PageCallResult {
|
|
112
119
|
status: PageCallStatus;
|
|
@@ -161,6 +168,51 @@ type MeasureElement = (el: Element) => DOMRect | null;
|
|
|
161
168
|
* unusual hosts (canvas overlays, iframes) can swap it.
|
|
162
169
|
*/
|
|
163
170
|
type HitTestPoint = (x: number, y: number) => Element | null;
|
|
171
|
+
/** A box in viewport px (same shape as the geometry helpers' `Box`). */
|
|
172
|
+
interface ViewportBox {
|
|
173
|
+
top: number;
|
|
174
|
+
left: number;
|
|
175
|
+
width: number;
|
|
176
|
+
height: number;
|
|
177
|
+
}
|
|
178
|
+
/** What {@link ClearTheWayHandler} gets: enough to move the agent's window off a target. */
|
|
179
|
+
interface ClearTheWayRequest {
|
|
180
|
+
/** The tool whose target is covered (for `select_tab`: the tab, or its drop-down). */
|
|
181
|
+
tool: 'highlight' | 'point' | 'scroll_to' | 'select_tab';
|
|
182
|
+
/** The target, as the result would name it. */
|
|
183
|
+
target: {
|
|
184
|
+
id?: string;
|
|
185
|
+
label: string;
|
|
186
|
+
};
|
|
187
|
+
/**
|
|
188
|
+
* The part of the target the user would see without the agent's UI in
|
|
189
|
+
* the way (`visibleWithoutAgent`), in viewport px. Move the window off
|
|
190
|
+
* this, keeping room for the effect's caption: an outline's caption sits
|
|
191
|
+
* up to 34 px above or below it, which `findClearPosition`'s default
|
|
192
|
+
* margin (`CLEAR_MARGIN`, 40) allows for.
|
|
193
|
+
*/
|
|
194
|
+
rect: ViewportBox;
|
|
195
|
+
/** The agent's UI over the target. */
|
|
196
|
+
cover: {
|
|
197
|
+
/** The outermost element matching `agentUiSelector` that was hit, when known. */
|
|
198
|
+
element: Element | null;
|
|
199
|
+
/** Box of the agent's UI over the target, in viewport px. */
|
|
200
|
+
rect: ViewportBox;
|
|
201
|
+
};
|
|
202
|
+
viewport: {
|
|
203
|
+
width: number;
|
|
204
|
+
height: number;
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Host hook called when the agent's own UI covers all or part of a
|
|
209
|
+
* `highlight`, `point`, `scroll_to` or `select_tab` target (a docked panel
|
|
210
|
+
* over one end of a heading counts). Move (or collapse) that UI, then return
|
|
211
|
+
* (or resolve to) `true` if you did something; anything else means "did
|
|
212
|
+
* nothing". Called at most once per call, given {@link CLEAR_THE_WAY_TIMEOUT_MS}
|
|
213
|
+
* to answer; errors count as "did nothing".
|
|
214
|
+
*/
|
|
215
|
+
type ClearTheWayHandler = (request: ClearTheWayRequest) => boolean | void | Promise<boolean | void>;
|
|
164
216
|
interface PageAgentConfig {
|
|
165
217
|
/** Called by the `navigate` tool. Without it, `navigate` is refused. */
|
|
166
218
|
navigate?: NavigateHandler;
|
|
@@ -168,6 +220,16 @@ interface PageAgentConfig {
|
|
|
168
220
|
aliasAttributes?: string[];
|
|
169
221
|
/** Host CSS selector for regions the agent must never see or point at (in addition to `[data-agent-ignore]`). */
|
|
170
222
|
ignoreSelector?: string;
|
|
223
|
+
/**
|
|
224
|
+
* Host CSS selector for the agent's own UI: its floating chat window,
|
|
225
|
+
* dock and launcher. Never read or pointed at (like `ignoreSelector`),
|
|
226
|
+
* and a target hidden under it is reported as `covered_by_agent` instead
|
|
227
|
+
* of `covered`. Not `[data-agent-ignore]`: that also marks sensitive page
|
|
228
|
+
* regions (an MFA dialog), which are not the agent's UI.
|
|
229
|
+
*/
|
|
230
|
+
agentUiSelector?: string;
|
|
231
|
+
/** Called when a target is covered by the agent's own UI; see {@link ClearTheWayHandler}. */
|
|
232
|
+
clearTheWay?: ClearTheWayHandler;
|
|
171
233
|
/** Layout reader; see {@link MeasureElement}. */
|
|
172
234
|
measure?: MeasureElement;
|
|
173
235
|
/** Hit test used to tell whether a target is covered; see {@link HitTestPoint}. */
|
|
@@ -179,6 +241,8 @@ interface ResolvedPageAgentConfig {
|
|
|
179
241
|
navigate: NavigateHandler | null;
|
|
180
242
|
aliasAttributes: string[];
|
|
181
243
|
ignoreSelector: string | null;
|
|
244
|
+
agentUiSelector: string | null;
|
|
245
|
+
clearTheWay: ClearTheWayHandler | null;
|
|
182
246
|
measure: MeasureElement | null;
|
|
183
247
|
hitTest: HitTestPoint | null;
|
|
184
248
|
tabSettleMs: number;
|
|
@@ -295,8 +359,25 @@ interface TargetVisibility {
|
|
|
295
359
|
*/
|
|
296
360
|
bounds: Box | null;
|
|
297
361
|
cut: CutSides;
|
|
298
|
-
/**
|
|
299
|
-
|
|
362
|
+
/**
|
|
363
|
+
* With `covered`: a modal dialog / inert page, another element on top, or
|
|
364
|
+
* the agent's own UI (see {@link VisibilityInput.coverAt}): the target
|
|
365
|
+
* would be at least partly visible if the agent's UI were not there.
|
|
366
|
+
*/
|
|
367
|
+
coveredBy?: 'modal' | 'element' | 'agent';
|
|
368
|
+
/**
|
|
369
|
+
* Set whenever the agent's own UI hides some of the target: all of it
|
|
370
|
+
* (`state: 'covered'`, `coveredBy: 'agent'`) or part of it (`visible` /
|
|
371
|
+
* `partial`: cut at a docked panel, or a window drawn over a corner).
|
|
372
|
+
* The box of the agent's UI over the target (the union of every agent-UI
|
|
373
|
+
* cover that overlaps it), in viewport px.
|
|
374
|
+
*/
|
|
375
|
+
agentCover?: Box;
|
|
376
|
+
/**
|
|
377
|
+
* Set with `agentCover`: the part of the target the user would see
|
|
378
|
+
* without the agent's UI in the way. Keep this clear to show all of it.
|
|
379
|
+
*/
|
|
380
|
+
visibleWithoutAgent?: Box;
|
|
300
381
|
/**
|
|
301
382
|
* Strips that cut the target (or a header-like cover over all of it), as
|
|
302
383
|
* measured: sticky headers and footers, docks, menus over one side.
|
|
@@ -313,9 +394,19 @@ interface VisibilityInput {
|
|
|
313
394
|
behindModal?: boolean;
|
|
314
395
|
/**
|
|
315
396
|
* Hit test: the box of whatever is drawn on top of the target at this
|
|
316
|
-
* viewport point, or null when the target itself (or nothing) is.
|
|
397
|
+
* viewport point, or null when the target itself (or nothing) is. Flag a
|
|
398
|
+
* cover that is the agent's own UI (its chat window, dock, launcher) with
|
|
399
|
+
* `agent: true`; a target covered only because of such covers is
|
|
400
|
+
* reported as `coveredBy: 'agent'`.
|
|
401
|
+
*/
|
|
402
|
+
coverAt?: (x: number, y: number) => CoverHit | null;
|
|
403
|
+
/**
|
|
404
|
+
* Hit test that looks through the agent's UI: what is on top of the
|
|
405
|
+
* target at this point once agent-UI covers are skipped. Used only to
|
|
406
|
+
* decide between `coveredBy: 'agent'` and `'element'`. Without it, a
|
|
407
|
+
* point under the agent's UI counts as uncovered there.
|
|
317
408
|
*/
|
|
318
|
-
|
|
409
|
+
coverBelowAgentAt?: (x: number, y: number) => CoverHit | null;
|
|
319
410
|
/**
|
|
320
411
|
* Sample grid per side (default 3, so 9 points), plus one probe just
|
|
321
412
|
* inside the middle of each edge of the visible part (see
|
|
@@ -323,6 +414,10 @@ interface VisibilityInput {
|
|
|
323
414
|
*/
|
|
324
415
|
grid?: number;
|
|
325
416
|
}
|
|
417
|
+
/** A cover box from a hit test; `agent` marks the agent's own UI. */
|
|
418
|
+
type CoverHit = RectLike & {
|
|
419
|
+
agent?: boolean;
|
|
420
|
+
};
|
|
326
421
|
/**
|
|
327
422
|
* Where the user can actually see a target: its box intersected with the
|
|
328
423
|
* viewport and every clipping ancestor, then hit-tested at a grid of points.
|
|
@@ -348,8 +443,54 @@ interface VisibilityInput {
|
|
|
348
443
|
* again, so a strip that only spans the part below a header is found too.
|
|
349
444
|
* Edge probes only find strips; they never count towards "most of it".
|
|
350
445
|
* 9 + 4 hit tests for an unobstructed target, 4 more per cut.
|
|
446
|
+
*
|
|
447
|
+
* Agent UI: when the target comes out `covered` and some covers were
|
|
448
|
+
* flagged `agent`, it is measured again looking through them
|
|
449
|
+
* (`coverBelowAgentAt`). If it would then be at least partly visible, the
|
|
450
|
+
* result is `coveredBy: 'agent'` with `agentCover` and
|
|
451
|
+
* `visibleWithoutAgent`; otherwise it stays `'element'` (something else
|
|
452
|
+
* covers it too, so moving the agent's UI would not help). A visible or
|
|
453
|
+
* partial target is drawn exactly as before (an agent panel over one side
|
|
454
|
+
* cuts it like any other strip), but when agent-UI covers overlap what it
|
|
455
|
+
* would show without them, it also gets `agentCover` and
|
|
456
|
+
* `visibleWithoutAgent`, so a host can move its UI off it.
|
|
351
457
|
*/
|
|
352
458
|
declare function computeVisibility(input: VisibilityInput): TargetVisibility;
|
|
459
|
+
/**
|
|
460
|
+
* Default gap {@link findClearPosition} keeps around the rect to keep
|
|
461
|
+
* clear: an outline's 4 px padding plus its 8 px gap and 22 px caption above
|
|
462
|
+
* or below the target, rounded up.
|
|
463
|
+
*/
|
|
464
|
+
declare const CLEAR_MARGIN = 40;
|
|
465
|
+
/**
|
|
466
|
+
* Whether two boxes overlap once `b` is grown by `margin` px on every side
|
|
467
|
+
* (touching edges do not count). For a docked panel: "would the target be
|
|
468
|
+
* clear if the panel were pinned to the other edge?" is
|
|
469
|
+
* `!overlaps(panelBoxAtOtherEdge, request.rect, CLEAR_MARGIN)`.
|
|
470
|
+
*/
|
|
471
|
+
declare function overlaps(a: RectLike, b: RectLike, margin?: number): boolean;
|
|
472
|
+
interface ClearPositionOptions {
|
|
473
|
+
/**
|
|
474
|
+
* Gap kept between the window and `keepClear`, in px. Default
|
|
475
|
+
* {@link CLEAR_MARGIN} (40): room for an outline's padding and its
|
|
476
|
+
* caption, drawn just above (or below) the target.
|
|
477
|
+
*/
|
|
478
|
+
margin?: number;
|
|
479
|
+
/** Gap kept between the window and the viewport edges, in px. Default 8. */
|
|
480
|
+
edge?: number;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Where to move a floating window (`windowRect`: the agent's chat window,
|
|
484
|
+
* say) so it no
|
|
485
|
+
* longer overlaps `keepClear` (the part of a target the user should see):
|
|
486
|
+
* the nearest top-left position, by straight-line distance from where the
|
|
487
|
+
* window is now, that keeps `margin` px between the two and `edge` px from
|
|
488
|
+
* the viewport edges. Returns the current position when the window already
|
|
489
|
+
* keeps clear (and fits; one that sticks out is moved back in), or null when no position fits (the window is too
|
|
490
|
+
* big for the room left beside, above and below `keepClear`). Pure; all
|
|
491
|
+
* boxes in viewport px.
|
|
492
|
+
*/
|
|
493
|
+
declare function findClearPosition(viewport: Size, windowRect: RectLike, keepClear: RectLike, opts?: ClearPositionOptions): Point | null;
|
|
353
494
|
|
|
354
495
|
declare const EFFECT_DEFAULT_MS = 8000;
|
|
355
496
|
declare const EFFECT_MIN_MS = 1000;
|
|
@@ -430,6 +571,8 @@ declare function nudgeIntoBounds(el: Element, visibility: TargetVisibility): boo
|
|
|
430
571
|
* control characters and malformed escapes.
|
|
431
572
|
*/
|
|
432
573
|
declare function parseAppPath(path: unknown): string | null;
|
|
574
|
+
/** How long `clearTheWay` has to answer before it counts as "did nothing". */
|
|
575
|
+
declare const CLEAR_THE_WAY_TIMEOUT_MS = 1000;
|
|
433
576
|
/**
|
|
434
577
|
* Run one tool call from the model. Accepts `unknown` on purpose: anything
|
|
435
578
|
* that is not `{ tool, input }` with a known tool and a valid input comes
|
|
@@ -616,4 +759,4 @@ declare function collapsedTabMenu(tab: Element): Element | null | undefined;
|
|
|
616
759
|
/** Back to a clean slate: default config, no effects. Registrations are kept (their owners destroy them). */
|
|
617
760
|
declare function resetPageAgent(): void;
|
|
618
761
|
|
|
619
|
-
export { type AgentTarget, type AgentTargetHandle, type AgentTargetKind, type Box, CALL_LABEL_MAX, CANDIDATES_MAX, type Candidate, type CutSides, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, type EffectKind, type EffectOutcome, type HighlightInput, type HighlightOptions, type HighlightStyle, type HitTestPoint, type JsonShape, LAYER_CLASS, LIST_MAX, type LabelPlacement, type ListTargetsInput, type ListTargetsOutput, type MeasureElement, NAVIGATE_PATH_MAX, type NavigateHandler, type NavigateInput, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, type PageAgentConfig, type PageCall, type PageCallResult, type PageCallStatus, type PageEffect, type PageToolDescription, type PageToolName, type Point, type PointInput, type PointOptions, type ReadPageTextOptions, type RenderCache, type ResolveFailure, type ResolveOptions, type ResolveResult, type ResolvedPageAgentConfig, type ScrollOutcome, type ScrollToInput, type SelectTabInput, type Size, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, type TabSummary, type TargetRef, type TargetSummary, type TargetVisibility, type ViewportSize, type VisibilityInput, type VisibilityState, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
|
762
|
+
export { type AgentTarget, type AgentTargetHandle, type AgentTargetKind, type Box, CALL_LABEL_MAX, CANDIDATES_MAX, CLEAR_MARGIN, CLEAR_THE_WAY_TIMEOUT_MS, type Candidate, type ClearPositionOptions, type ClearTheWayHandler, type ClearTheWayRequest, type CoverHit, type CoveredReason, type CutSides, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, type EffectKind, type EffectOutcome, type HighlightInput, type HighlightOptions, type HighlightStyle, type HitTestPoint, type JsonShape, LAYER_CLASS, LIST_MAX, type LabelPlacement, type ListTargetsInput, type ListTargetsOutput, type MeasureElement, NAVIGATE_PATH_MAX, type NavigateHandler, type NavigateInput, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, type PageAgentConfig, type PageCall, type PageCallResult, type PageCallStatus, type PageEffect, type PageToolDescription, type PageToolName, type Point, type PointInput, type PointOptions, type ReadPageTextOptions, type RenderCache, type ResolveFailure, type ResolveOptions, type ResolveResult, type ResolvedPageAgentConfig, type ScrollOutcome, type ScrollToInput, type SelectTabInput, type Size, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, type TabSummary, type TargetRef, type TargetSummary, type TargetVisibility, type ViewportBox, type ViewportSize, type VisibilityInput, type VisibilityState, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, findClearPosition, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, overlaps, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,8 @@ var defaults = () => ({
|
|
|
6
6
|
navigate: null,
|
|
7
7
|
aliasAttributes: [],
|
|
8
8
|
ignoreSelector: null,
|
|
9
|
+
agentUiSelector: null,
|
|
10
|
+
clearTheWay: null,
|
|
9
11
|
measure: null,
|
|
10
12
|
hitTest: null,
|
|
11
13
|
tabSettleMs: DEFAULT_TAB_SETTLE_MS
|
|
@@ -25,6 +27,11 @@ function configurePageAgent(next) {
|
|
|
25
27
|
if ("ignoreSelector" in next) {
|
|
26
28
|
merged.ignoreSelector = validSelector(next.ignoreSelector) ? next.ignoreSelector.trim() : null;
|
|
27
29
|
}
|
|
30
|
+
if ("agentUiSelector" in next) {
|
|
31
|
+
merged.agentUiSelector = validSelector(next.agentUiSelector) ? next.agentUiSelector.trim() : null;
|
|
32
|
+
}
|
|
33
|
+
if ("clearTheWay" in next)
|
|
34
|
+
merged.clearTheWay = typeof next.clearTheWay === "function" ? next.clearTheWay : null;
|
|
28
35
|
if ("measure" in next) merged.measure = typeof next.measure === "function" ? next.measure : null;
|
|
29
36
|
if ("hitTest" in next) merged.hitTest = typeof next.hitTest === "function" ? next.hitTest : null;
|
|
30
37
|
if (next.tabSettleMs !== void 0 && Number.isFinite(next.tabSettleMs)) {
|
|
@@ -56,15 +63,22 @@ var BUILT_IN_IGNORE = `[data-agent-ignore],.${LAYER_CLASS}`;
|
|
|
56
63
|
var hasDom = () => typeof document !== "undefined";
|
|
57
64
|
function isIgnored(el, extraSelector) {
|
|
58
65
|
if (el.closest(BUILT_IN_IGNORE)) return true;
|
|
59
|
-
const
|
|
60
|
-
return safeClosest(el,
|
|
66
|
+
const { ignoreSelector, agentUiSelector } = getPageAgentConfig();
|
|
67
|
+
return safeClosest(el, ignoreSelector) !== null || safeClosest(el, agentUiSelector) !== null || safeClosest(el, extraSelector ?? null) !== null;
|
|
68
|
+
}
|
|
69
|
+
function agentUiRoot(el) {
|
|
70
|
+
const selector = getPageAgentConfig().agentUiSelector;
|
|
71
|
+
let root = null;
|
|
72
|
+
for (let node = safeClosest(el, selector); node; node = safeClosest(node.parentElement, selector))
|
|
73
|
+
root = node;
|
|
74
|
+
return root;
|
|
61
75
|
}
|
|
62
76
|
function safeClosest(el, selector) {
|
|
63
|
-
if (!selector) return
|
|
77
|
+
if (!selector || !el) return null;
|
|
64
78
|
try {
|
|
65
|
-
return el.closest(selector)
|
|
79
|
+
return el.closest(selector);
|
|
66
80
|
} catch {
|
|
67
|
-
return
|
|
81
|
+
return null;
|
|
68
82
|
}
|
|
69
83
|
}
|
|
70
84
|
function defaultMeasure(el) {
|
|
@@ -233,6 +247,38 @@ function containsBox(outer, inner) {
|
|
|
233
247
|
var containsPoint = (box, x, y) => x >= box.left && x <= box.left + box.width && y >= box.top && y <= box.top + box.height;
|
|
234
248
|
var NO_CUT = { top: false, right: false, bottom: false, left: false };
|
|
235
249
|
function computeVisibility(input) {
|
|
250
|
+
const coverAt = input.coverAt;
|
|
251
|
+
if (!coverAt) return measureOnce(input, void 0);
|
|
252
|
+
const agentBoxes = [];
|
|
253
|
+
const first = measureOnce(input, (x, y) => {
|
|
254
|
+
const hit = coverAt(x, y);
|
|
255
|
+
if (hit?.agent) {
|
|
256
|
+
const box = toBox(hit);
|
|
257
|
+
if (containsPoint(box, x, y) && !agentBoxes.some((b) => sameBox(b, box)))
|
|
258
|
+
agentBoxes.push(box);
|
|
259
|
+
}
|
|
260
|
+
return hit;
|
|
261
|
+
});
|
|
262
|
+
const fullyCovered = first.state === "covered" && first.coveredBy === "element";
|
|
263
|
+
const onScreen2 = first.state === "visible" || first.state === "partial";
|
|
264
|
+
if (agentBoxes.length === 0 || !fullyCovered && !onScreen2) return first;
|
|
265
|
+
const below = input.coverBelowAgentAt ?? coverAt;
|
|
266
|
+
const without = measureOnce(input, (x, y) => {
|
|
267
|
+
const hit = below(x, y);
|
|
268
|
+
return hit && !hit.agent ? hit : null;
|
|
269
|
+
});
|
|
270
|
+
const clear = without.visible;
|
|
271
|
+
if (!clear) return first;
|
|
272
|
+
const over = agentBoxes.filter((b) => intersectBoxes(b, clear));
|
|
273
|
+
if (onScreen2 && over.length === 0) return first;
|
|
274
|
+
return {
|
|
275
|
+
...first,
|
|
276
|
+
...fullyCovered ? { coveredBy: "agent" } : {},
|
|
277
|
+
agentCover: unionRects(over.length ? over : agentBoxes),
|
|
278
|
+
visibleWithoutAgent: clear
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
function measureOnce(input, coverAt) {
|
|
236
282
|
if (!input.rect) {
|
|
237
283
|
return { state: "unrendered", rect: null, visible: null, bounds: null, cut: NO_CUT, bands: [] };
|
|
238
284
|
}
|
|
@@ -259,14 +305,14 @@ function computeVisibility(input) {
|
|
|
259
305
|
return result("clipped", null);
|
|
260
306
|
}
|
|
261
307
|
const bands = [];
|
|
262
|
-
if (
|
|
308
|
+
if (coverAt) {
|
|
263
309
|
const n = Math.min(Math.max(Math.round(input.grid ?? 3), 1), 5);
|
|
264
310
|
const samples = [];
|
|
265
311
|
for (let row = 0; row < n; row++) {
|
|
266
312
|
for (let col = 0; col < n; col++) {
|
|
267
313
|
const x = visible.left + visible.width * (col + 0.5) / n;
|
|
268
314
|
const y = visible.top + visible.height * (row + 0.5) / n;
|
|
269
|
-
const hit =
|
|
315
|
+
const hit = coverAt(x, y);
|
|
270
316
|
const cover = hit ? toBox(hit) : null;
|
|
271
317
|
samples.push({ x, y, cover: cover && containsPoint(cover, x, y) ? cover : null });
|
|
272
318
|
}
|
|
@@ -282,7 +328,7 @@ function computeVisibility(input) {
|
|
|
282
328
|
const key = `${Math.round(x * 2)},${Math.round(y * 2)}`;
|
|
283
329
|
if (probed.has(key)) continue;
|
|
284
330
|
probed.add(key);
|
|
285
|
-
const hit =
|
|
331
|
+
const hit = coverAt(x, y);
|
|
286
332
|
const cover = hit ? toBox(hit) : null;
|
|
287
333
|
addCover(cover && containsPoint(cover, x, y) ? cover : null);
|
|
288
334
|
}
|
|
@@ -374,6 +420,41 @@ function trimBand(visible, cover) {
|
|
|
374
420
|
if (spansY && cRight >= vRight - EDGE) return trim("right", cover.left);
|
|
375
421
|
return null;
|
|
376
422
|
}
|
|
423
|
+
var CLEAR_MARGIN = 40;
|
|
424
|
+
function overlaps(a, b, margin = 0) {
|
|
425
|
+
return intersectBoxes(a, inflate(b, Math.max(0, margin))) !== null;
|
|
426
|
+
}
|
|
427
|
+
function findClearPosition(viewport, windowRect, keepClear, opts = {}) {
|
|
428
|
+
const margin = Math.max(0, opts.margin ?? CLEAR_MARGIN);
|
|
429
|
+
const edge = Math.max(0, opts.edge ?? 8);
|
|
430
|
+
const w = windowRect.width;
|
|
431
|
+
const h = windowRect.height;
|
|
432
|
+
const minX = edge;
|
|
433
|
+
const minY = edge;
|
|
434
|
+
const maxX = viewport.width - edge - w;
|
|
435
|
+
const maxY = viewport.height - edge - h;
|
|
436
|
+
if (maxX < minX || maxY < minY) return null;
|
|
437
|
+
const keep = inflate(keepClear, margin);
|
|
438
|
+
const clampX = (x) => Math.min(Math.max(x, minX), maxX);
|
|
439
|
+
const clampY = (y) => Math.min(Math.max(y, minY), maxY);
|
|
440
|
+
const overlaps2 = (left, top) => intersectBoxes({ left, top, width: w, height: h }, keep) !== null;
|
|
441
|
+
const here = { left: windowRect.left, top: windowRect.top };
|
|
442
|
+
const keepRight = keep.left + keep.width;
|
|
443
|
+
const keepBottom = keep.top + keep.height;
|
|
444
|
+
const candidates = [
|
|
445
|
+
{ left: clampX(here.left), top: clampY(here.top) },
|
|
446
|
+
{ left: keep.left - w, top: clampY(here.top) },
|
|
447
|
+
{ left: keepRight, top: clampY(here.top) },
|
|
448
|
+
{ left: clampX(here.left), top: keep.top - h },
|
|
449
|
+
{ left: clampX(here.left), top: keepBottom }
|
|
450
|
+
].filter(
|
|
451
|
+
(c) => c.left >= minX && c.left <= maxX && c.top >= minY && c.top <= maxY && !overlaps2(c.left, c.top)
|
|
452
|
+
);
|
|
453
|
+
if (candidates.length === 0) return null;
|
|
454
|
+
const distance = (c) => Math.hypot(c.left - here.left, c.top - here.top);
|
|
455
|
+
const best = candidates.reduce((a, b) => distance(b) < distance(a) ? b : a);
|
|
456
|
+
return { top: Math.round(best.top), left: Math.round(best.left) };
|
|
457
|
+
}
|
|
377
458
|
|
|
378
459
|
// src/registry.ts
|
|
379
460
|
var TARGET_LABEL_MAX = 80;
|
|
@@ -647,10 +728,12 @@ function quote(value) {
|
|
|
647
728
|
// src/visibility.ts
|
|
648
729
|
var CLIPS = /^(hidden|clip|auto|scroll|overlay)$/;
|
|
649
730
|
var COVER_WALK_MAX = 16;
|
|
650
|
-
function defaultHitTest(x, y) {
|
|
731
|
+
function defaultHitTest(x, y, skipAgentUi = false) {
|
|
651
732
|
if (typeof document.elementsFromPoint !== "function") return null;
|
|
652
733
|
for (const el of document.elementsFromPoint(x, y)) {
|
|
653
|
-
if (
|
|
734
|
+
if (el.closest(`.${LAYER_CLASS}`)) continue;
|
|
735
|
+
if (skipAgentUi && agentUiRoot(el)) continue;
|
|
736
|
+
return el;
|
|
654
737
|
}
|
|
655
738
|
return null;
|
|
656
739
|
}
|
|
@@ -662,6 +745,13 @@ function hitTest(x, y) {
|
|
|
662
745
|
return null;
|
|
663
746
|
}
|
|
664
747
|
}
|
|
748
|
+
function hitTestBelowAgentUi(x, y) {
|
|
749
|
+
try {
|
|
750
|
+
return defaultHitTest(x, y, true);
|
|
751
|
+
} catch {
|
|
752
|
+
return null;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
665
755
|
var toBox2 = (r) => ({ top: r.top, left: r.left, width: r.width, height: r.height });
|
|
666
756
|
function clippingBoxes(el) {
|
|
667
757
|
const view = el.ownerDocument.defaultView;
|
|
@@ -710,35 +800,56 @@ function isBehindModal(el) {
|
|
|
710
800
|
const top = modals[modals.length - 1];
|
|
711
801
|
return !!top && !top.contains(el);
|
|
712
802
|
}
|
|
713
|
-
function coverBox(el,
|
|
714
|
-
const hit = hitTest(x, y);
|
|
803
|
+
function coverBox(el, hit, onAgent) {
|
|
715
804
|
if (!hit || hit === el || el.contains(hit) || hit.contains(el)) return null;
|
|
716
805
|
let box = layoutRect(hit);
|
|
717
806
|
if (!box) return null;
|
|
718
|
-
|
|
807
|
+
const agent = agentUiRoot(hit);
|
|
808
|
+
let node = hit === agent ? null : hit.parentElement;
|
|
719
809
|
for (let i = 0; node && i < COVER_WALK_MAX && !node.contains(el); i++) {
|
|
720
810
|
const outer = layoutRect(node);
|
|
721
811
|
if (!outer || !encloses(outer, box)) break;
|
|
722
812
|
box = outer;
|
|
813
|
+
if (node === agent) break;
|
|
723
814
|
node = node.parentElement;
|
|
724
815
|
}
|
|
725
|
-
return box;
|
|
816
|
+
if (!agent) return box;
|
|
817
|
+
onAgent?.(agent);
|
|
818
|
+
return { top: box.top, left: box.left, width: box.width, height: box.height, agent: true };
|
|
726
819
|
}
|
|
727
820
|
function encloses(outer, inner) {
|
|
728
821
|
return outer.top <= inner.top + 1 && outer.left <= inner.left + 1 && outer.bottom >= inner.bottom - 1 && outer.right >= inner.right - 1;
|
|
729
822
|
}
|
|
730
823
|
function measureVisibility(el) {
|
|
824
|
+
return measureVisibilityDetail(el).visibility;
|
|
825
|
+
}
|
|
826
|
+
function measureVisibilityDetail(el) {
|
|
731
827
|
const rect = hasDom() && el.isConnected ? layoutRect(el) : null;
|
|
732
828
|
if (!rect) {
|
|
733
|
-
return
|
|
829
|
+
return {
|
|
830
|
+
visibility: computeVisibility({ rect: null, viewport: viewportSize() }),
|
|
831
|
+
agentElement: null
|
|
832
|
+
};
|
|
734
833
|
}
|
|
735
|
-
|
|
834
|
+
const agents = [];
|
|
835
|
+
const onAgent = (root) => {
|
|
836
|
+
if (!agents.includes(root)) agents.push(root);
|
|
837
|
+
};
|
|
838
|
+
const config2 = getPageAgentConfig();
|
|
839
|
+
const visibility = computeVisibility({
|
|
736
840
|
rect,
|
|
737
841
|
viewport: viewportSize(),
|
|
738
842
|
clips: clippingBoxes(el),
|
|
739
843
|
behindModal: isBehindModal(el),
|
|
740
|
-
coverAt: (x, y) => coverBox(el, x, y)
|
|
844
|
+
coverAt: (x, y) => coverBox(el, hitTest(x, y), onAgent),
|
|
845
|
+
// A custom hit test cannot look through the agent's UI: its agent hits
|
|
846
|
+
// then count as uncovered points (see computeVisibility).
|
|
847
|
+
...config2.agentUiSelector && !config2.hitTest ? { coverBelowAgentAt: (x, y) => coverBox(el, hitTestBelowAgentUi(x, y)) } : {}
|
|
741
848
|
});
|
|
849
|
+
return {
|
|
850
|
+
visibility,
|
|
851
|
+
agentElement: visibility.agentCover ? agents[0] ?? null : null
|
|
852
|
+
};
|
|
742
853
|
}
|
|
743
854
|
|
|
744
855
|
// src/controller.ts
|
|
@@ -1334,11 +1445,12 @@ var MENU_NOTE = "The tabs are shown as a drop-down menu here (the page is narrow
|
|
|
1334
1445
|
var wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1335
1446
|
var SCROLL_SETTLE_MAX_MS = 1e3;
|
|
1336
1447
|
var SCROLL_SETTLE_STEP_MS = 50;
|
|
1337
|
-
|
|
1338
|
-
|
|
1448
|
+
var CLEAR_THE_WAY_TIMEOUT_MS = 1e3;
|
|
1449
|
+
async function settleLayout(...els) {
|
|
1450
|
+
const key = () => els.map((el) => {
|
|
1339
1451
|
const r = layoutRect(el);
|
|
1340
1452
|
return r ? `${Math.round(r.top)},${Math.round(r.left)}` : "";
|
|
1341
|
-
};
|
|
1453
|
+
}).join(";");
|
|
1342
1454
|
let last = key();
|
|
1343
1455
|
let stable = 0;
|
|
1344
1456
|
for (let waited = 0; waited < SCROLL_SETTLE_MAX_MS; waited += SCROLL_SETTLE_STEP_MS) {
|
|
@@ -1355,25 +1467,78 @@ async function settleLayout(el) {
|
|
|
1355
1467
|
async function reveal(el, scrolledAlready) {
|
|
1356
1468
|
let scrolled = scrolledAlready ?? scrollIntoContainers(el);
|
|
1357
1469
|
if (scrolled) await settleLayout(el);
|
|
1358
|
-
let
|
|
1359
|
-
if (visibility.bands.length > 0 && nudgeIntoBounds(el, visibility)) {
|
|
1470
|
+
let measured = measureVisibilityDetail(el);
|
|
1471
|
+
if (measured.visibility.bands.length > 0 && nudgeIntoBounds(el, measured.visibility)) {
|
|
1360
1472
|
scrolled = true;
|
|
1361
1473
|
await settleLayout(el);
|
|
1362
|
-
|
|
1474
|
+
measured = measureVisibilityDetail(el);
|
|
1475
|
+
}
|
|
1476
|
+
return { scrolled, ...measured };
|
|
1477
|
+
}
|
|
1478
|
+
function within(promise, ms, fallback) {
|
|
1479
|
+
return new Promise((resolve) => {
|
|
1480
|
+
const timer = setTimeout(() => resolve(fallback), ms);
|
|
1481
|
+
promise.then(
|
|
1482
|
+
(value) => {
|
|
1483
|
+
clearTimeout(timer);
|
|
1484
|
+
resolve(value);
|
|
1485
|
+
},
|
|
1486
|
+
() => {
|
|
1487
|
+
clearTimeout(timer);
|
|
1488
|
+
resolve(fallback);
|
|
1489
|
+
}
|
|
1490
|
+
);
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1493
|
+
async function askToClearTheWay(tool, resolved, visibility, agentElement) {
|
|
1494
|
+
const handler = getPageAgentConfig().clearTheWay;
|
|
1495
|
+
const { agentCover, visibleWithoutAgent } = visibility;
|
|
1496
|
+
if (!handler || !agentCover || !visibleWithoutAgent) return false;
|
|
1497
|
+
const request = {
|
|
1498
|
+
tool,
|
|
1499
|
+
target: summary(resolved),
|
|
1500
|
+
rect: { ...visibleWithoutAgent },
|
|
1501
|
+
cover: { element: agentElement, rect: { ...agentCover } },
|
|
1502
|
+
viewport: viewportSize()
|
|
1503
|
+
};
|
|
1504
|
+
let answer;
|
|
1505
|
+
try {
|
|
1506
|
+
answer = handler(request);
|
|
1507
|
+
} catch {
|
|
1508
|
+
return false;
|
|
1363
1509
|
}
|
|
1364
|
-
return
|
|
1510
|
+
return await within(Promise.resolve(answer), CLEAR_THE_WAY_TIMEOUT_MS, false) === true;
|
|
1511
|
+
}
|
|
1512
|
+
async function revealForCall(tool, resolved, scrolledAlready) {
|
|
1513
|
+
const { scrolled, visibility, agentElement } = await reveal(resolved.el, scrolledAlready);
|
|
1514
|
+
if (!visibility.agentCover) return { scrolled, visibility, clearedTheWay: false };
|
|
1515
|
+
if (!await askToClearTheWay(tool, resolved, visibility, agentElement)) {
|
|
1516
|
+
return { scrolled, visibility, clearedTheWay: false };
|
|
1517
|
+
}
|
|
1518
|
+
await settleLayout(...agentElement ? [resolved.el, agentElement] : [resolved.el]);
|
|
1519
|
+
return {
|
|
1520
|
+
scrolled,
|
|
1521
|
+
visibility: measureVisibilityDetail(resolved.el).visibility,
|
|
1522
|
+
clearedTheWay: true
|
|
1523
|
+
};
|
|
1365
1524
|
}
|
|
1366
1525
|
var onScreen = (v) => v.state === "visible" || v.state === "partial";
|
|
1526
|
+
var COVER_REASON = {
|
|
1527
|
+
modal: "behind_dialog",
|
|
1528
|
+
element: "covered",
|
|
1529
|
+
agent: "covered_by_agent"
|
|
1530
|
+
};
|
|
1367
1531
|
function coveredResult(visibility, label2) {
|
|
1368
1532
|
if (visibility.state !== "covered") return null;
|
|
1369
1533
|
const what = `"${label2.replace(/"/g, "'")}"`;
|
|
1370
|
-
const
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
message: modal ? `${what} is behind a dialog right now, so it cannot be shown. Point at something in the dialog, or ask the user to close it.` : `${what} is covered by another part of the page (such as a header, panel or popup), so it cannot be shown right now.`,
|
|
1374
|
-
output: { reason: modal ? "behind_dialog" : "covered" }
|
|
1375
|
-
};
|
|
1534
|
+
const reason = COVER_REASON[visibility.coveredBy ?? "element"];
|
|
1535
|
+
const message = reason === "behind_dialog" ? `${what} is behind a dialog right now, so it cannot be shown. Point at something in the dialog, or ask the user to close it.` : reason === "covered_by_agent" ? `${what} is covered by the assistant's own window, so it cannot be shown right now. Ask the user to move or minimise the assistant window, then try again.` : `${what} is covered by another part of the page (such as a header, panel or popup), so it cannot be shown right now.`;
|
|
1536
|
+
return { status: "hidden", message, output: { reason } };
|
|
1376
1537
|
}
|
|
1538
|
+
var agentNotes = (clearedTheWay, visibility) => ({
|
|
1539
|
+
...clearedTheWay ? { clearedTheWay: true } : {},
|
|
1540
|
+
...visibility.agentCover && onScreen(visibility) ? { partlyCoveredByAgent: true } : {}
|
|
1541
|
+
});
|
|
1377
1542
|
async function confirmSelected(el, settleMs) {
|
|
1378
1543
|
const selected = () => el.isConnected && el.getAttribute("aria-selected") === "true";
|
|
1379
1544
|
await Promise.resolve();
|
|
@@ -1420,7 +1585,7 @@ var handlers = {
|
|
|
1420
1585
|
const durationMs2 = optionalDuration(input.durationMs);
|
|
1421
1586
|
const resolved = resolveTarget(target2);
|
|
1422
1587
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1423
|
-
const { scrolled, visibility } = await
|
|
1588
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall("highlight", resolved);
|
|
1424
1589
|
const covered = coveredResult(visibility, resolved.label);
|
|
1425
1590
|
if (covered) return covered;
|
|
1426
1591
|
if (!isRendered(resolved.el)) {
|
|
@@ -1440,6 +1605,7 @@ var handlers = {
|
|
|
1440
1605
|
style,
|
|
1441
1606
|
scrolled,
|
|
1442
1607
|
inViewport: onScreen(visibility),
|
|
1608
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1443
1609
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1444
1610
|
}
|
|
1445
1611
|
};
|
|
@@ -1451,7 +1617,7 @@ var handlers = {
|
|
|
1451
1617
|
const durationMs2 = optionalDuration(input.durationMs);
|
|
1452
1618
|
const resolved = resolveTarget(target2);
|
|
1453
1619
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1454
|
-
const { scrolled, visibility } = await
|
|
1620
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall("point", resolved);
|
|
1455
1621
|
const covered = coveredResult(visibility, resolved.label);
|
|
1456
1622
|
if (covered) return covered;
|
|
1457
1623
|
if (!isRendered(resolved.el)) {
|
|
@@ -1470,6 +1636,7 @@ var handlers = {
|
|
|
1470
1636
|
target: summary(resolved),
|
|
1471
1637
|
scrolled,
|
|
1472
1638
|
inViewport: onScreen(visibility),
|
|
1639
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1473
1640
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1474
1641
|
}
|
|
1475
1642
|
};
|
|
@@ -1481,13 +1648,19 @@ var handlers = {
|
|
|
1481
1648
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1482
1649
|
const outcome = scrollToTarget(resolved.el);
|
|
1483
1650
|
if (outcome.status !== "ok") return failure(outcome);
|
|
1484
|
-
const { scrolled, visibility } = await
|
|
1651
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall(
|
|
1652
|
+
"scroll_to",
|
|
1653
|
+
resolved,
|
|
1654
|
+
outcome.scrolled
|
|
1655
|
+
);
|
|
1656
|
+
if (visibility.coveredBy === "agent") return coveredResult(visibility, resolved.label);
|
|
1485
1657
|
return {
|
|
1486
1658
|
status: "ok",
|
|
1487
1659
|
output: {
|
|
1488
1660
|
target: summary(resolved),
|
|
1489
1661
|
scrolled,
|
|
1490
1662
|
inViewport: onScreen(visibility),
|
|
1663
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1491
1664
|
...visibility.state === "covered" ? { covered: true } : {},
|
|
1492
1665
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1493
1666
|
}
|
|
@@ -1526,6 +1699,8 @@ var handlers = {
|
|
|
1526
1699
|
if (isBehindModal(shown)) {
|
|
1527
1700
|
return { status: "hidden", message: "That tab is behind a dialog right now." };
|
|
1528
1701
|
}
|
|
1702
|
+
const measured = measureVisibilityDetail(shown);
|
|
1703
|
+
const clearedTheWay = !!measured.visibility.agentCover && await askToClearTheWay("select_tab", resolved, measured.visibility, measured.agentElement);
|
|
1529
1704
|
el.click();
|
|
1530
1705
|
if (!await confirmSelected(el, getPageAgentConfig().tabSettleMs)) {
|
|
1531
1706
|
return { status: "error", message: "The tab did not become selected." };
|
|
@@ -1533,7 +1708,12 @@ var handlers = {
|
|
|
1533
1708
|
return {
|
|
1534
1709
|
status: "ok",
|
|
1535
1710
|
...asMenu ? { message: `${MENU_NOTE} The menu now shows this tab.` } : {},
|
|
1536
|
-
output: {
|
|
1711
|
+
output: {
|
|
1712
|
+
target: summary(resolved),
|
|
1713
|
+
alreadySelected: false,
|
|
1714
|
+
...extra,
|
|
1715
|
+
...clearedTheWay ? { clearedTheWay: true } : {}
|
|
1716
|
+
}
|
|
1537
1717
|
};
|
|
1538
1718
|
},
|
|
1539
1719
|
async navigate(raw) {
|
|
@@ -1574,4 +1754,4 @@ function resetPageAgent() {
|
|
|
1574
1754
|
resetPageAgentConfig();
|
|
1575
1755
|
}
|
|
1576
1756
|
|
|
1577
|
-
export { CALL_LABEL_MAX, CANDIDATES_MAX, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, LAYER_CLASS, LIST_MAX, NAVIGATE_PATH_MAX, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
|
1757
|
+
export { CALL_LABEL_MAX, CANDIDATES_MAX, CLEAR_MARGIN, CLEAR_THE_WAY_TIMEOUT_MS, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, LAYER_CLASS, LIST_MAX, NAVIGATE_PATH_MAX, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, findClearPosition, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, overlaps, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@classic-homes/page-agent-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Framework-agnostic page layer for an in-app AI agent: named targets, safe target resolution, a page snapshot, highlight / spotlight / pointer effects and one schema-validated executor for model tool calls",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "dist/index.js",
|