@classic-homes/page-agent-core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -12
- package/dist/index.d.ts +181 -18
- package/dist/index.js +248 -38
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,20 +43,30 @@ durationMs })`, `point(ref, { label, durationMs })`, `scrollToTarget(ref)`,
|
|
|
43
43
|
`attachPageAgent()` wires Escape and `popstate` to clear; call
|
|
44
44
|
`notifyNavigated()` after SPA navigation.
|
|
45
45
|
- **Executor**: `configurePageAgent({ navigate, aliasAttributes,
|
|
46
|
-
ignoreSelector })` and `executePageCall(call)`, the single entry point for
|
|
46
|
+
ignoreSelector, agentUiSelector, clearTheWay })` and `executePageCall(call)`, the single entry point for
|
|
47
47
|
the model's tool calls. `PAGE_TOOL_NAMES` and `PAGE_TOOLS` (plain-JSON
|
|
48
48
|
input shapes) let the agent server mirror the contract exactly.
|
|
49
49
|
- **Visibility**: `measureVisibility(el)` says what of a target the user can
|
|
50
|
-
see: clipped to the viewport and its scroll boxes, cut where a
|
|
51
|
-
header / footer / dock
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
see: clipped to the viewport and its scroll boxes, cut where a strip
|
|
51
|
+
covers one side of it (a sticky header / footer / dock, or a menu over one
|
|
52
|
+
end: a cover spanning the target's visible part along one axis and
|
|
53
|
+
reaching past an edge on the other, viewport-wide or not; cuts combine),
|
|
54
|
+
drawn over smaller covers (hit-tested with `document.elementsFromPoint`
|
|
55
|
+
at a 3 x 3 grid plus a probe inside each edge, the layer skipped), and
|
|
56
|
+
`covered` under a cover that hides most of it, behind a modal dialog or in
|
|
57
|
+
`inert`. Bindings draw only `visible` and hide (not
|
|
54
58
|
remove) effects while it is null. The maths is the pure
|
|
55
59
|
`computeVisibility({ rect, viewport, clips, behindModal, coverAt })`.
|
|
60
|
+
A target hidden only by the agent's own UI (`agentUiSelector`) is
|
|
61
|
+
`coveredBy: 'agent'`; whenever that UI hides all or part of it, the
|
|
62
|
+
result carries `agentCover` and `visibleWithoutAgent`.
|
|
56
63
|
- **Collapsed tab rows**: `collapsedTabMenu(tab)` and the
|
|
57
64
|
`data-tablist-menu` DOM contract (below).
|
|
58
65
|
- **Geometry**: `unionRects`, `inflate`, `clampToViewport`, `intersectBoxes`,
|
|
59
|
-
`placeLabel` (with optional `bounds`), `pointerTarget` for the UI layer
|
|
66
|
+
`placeLabel` (with optional `bounds`), `pointerTarget` for the UI layer,
|
|
67
|
+
and `findClearPosition(viewport, windowRect, keepClear, { margin, edge })`,
|
|
68
|
+
`overlaps(a, b, margin)` and `CLEAR_MARGIN` for hosts moving their chat
|
|
69
|
+
window or dock off a target (below).
|
|
60
70
|
|
|
61
71
|
```ts
|
|
62
72
|
import {
|
|
@@ -101,8 +111,110 @@ output? }` with `status` one of `ok`, `not_found`, `ambiguous`, `hidden`,
|
|
|
101
111
|
`highlight`, `point` and `scroll_to` wait for their scroll to settle, scroll
|
|
102
112
|
once more if a sticky header or footer still covers the target, and measure
|
|
103
113
|
`inViewport` after that. `highlight` / `point` on a target the user cannot
|
|
104
|
-
see because something covers it return `hidden` with `output.reason`
|
|
105
|
-
`
|
|
114
|
+
see because something covers it return `hidden` with `output.reason` (type
|
|
115
|
+
`CoveredReason`, exactly these three):
|
|
116
|
+
|
|
117
|
+
| `output.reason` | Covered by |
|
|
118
|
+
| ------------------ | ----------------------------------------------------------------------------- |
|
|
119
|
+
| `behind_dialog` | A modal dialog, or the target is in an `inert` subtree |
|
|
120
|
+
| `covered` | Another part of the page (a header, panel, popup) that scrolling cannot clear |
|
|
121
|
+
| `covered_by_agent` | The agent's own UI (`agentUiSelector`): its chat window, dock or launcher |
|
|
122
|
+
|
|
123
|
+
`scroll_to` also returns `hidden` / `covered_by_agent` when its target ends up
|
|
124
|
+
wholly under the agent's UI; under any other cover it still answers `ok` with
|
|
125
|
+
`covered: true`. A target only partly under the agent's UI is `ok`, with
|
|
126
|
+
`output.partlyCoveredByAgent: true`.
|
|
127
|
+
|
|
128
|
+
## The agent's own UI
|
|
129
|
+
|
|
130
|
+
The agent's chat window, or a panel docked to one edge that overlaps the
|
|
131
|
+
page, can sit over the very thing the user asked about. Tell the core which
|
|
132
|
+
elements are that UI; it then sees beneath it, says when it is in the way,
|
|
133
|
+
and lets you move it:
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
import {
|
|
137
|
+
configurePageAgent,
|
|
138
|
+
findClearPosition,
|
|
139
|
+
overlaps,
|
|
140
|
+
viewportSize,
|
|
141
|
+
CLEAR_MARGIN,
|
|
142
|
+
type ClearTheWayHandler,
|
|
143
|
+
} from '@classic-homes/page-agent-core';
|
|
144
|
+
|
|
145
|
+
// Your chat: floatAt(point), setDockEdge('left' | 'right'), and its state.
|
|
146
|
+
const clearTheWay: ClearTheWayHandler = ({ rect, cover }) => {
|
|
147
|
+
const vp = viewportSize();
|
|
148
|
+
if (chat.mode === 'dock') {
|
|
149
|
+
// Would the target be clear with the dock on the other edge?
|
|
150
|
+
const other = chat.edge === 'left' ? 'right' : 'left';
|
|
151
|
+
const there = {
|
|
152
|
+
top: 0,
|
|
153
|
+
height: vp.height,
|
|
154
|
+
width: chat.dockWidth,
|
|
155
|
+
left: other === 'left' ? 0 : vp.width - chat.dockWidth,
|
|
156
|
+
};
|
|
157
|
+
if (!overlaps(there, rect, CLEAR_MARGIN)) {
|
|
158
|
+
chat.setDockEdge(other);
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// Floating (or no free edge: turn the dock into the floating window).
|
|
163
|
+
const size = chat.floatSize; // { width, height } of the floating window
|
|
164
|
+
const from = chat.mode === 'dock' ? { ...cover.rect, ...size } : cover.rect;
|
|
165
|
+
const to = findClearPosition(vp, from, rect); // default margin: CLEAR_MARGIN
|
|
166
|
+
if (!to) return false; // nowhere to go: the effect is cut, or covered_by_agent
|
|
167
|
+
chat.floatAt(to);
|
|
168
|
+
return true;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
configurePageAgent({ agentUiSelector: '[data-assistant-ui]', clearTheWay });
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
- `agentUiSelector` (default none) is never read or pointed at, like
|
|
175
|
+
`ignoreSelector`. It is a separate option because neither
|
|
176
|
+
`ignoreSelector` nor `[data-agent-ignore]` means "the agent's UI":
|
|
177
|
+
`data-agent-ignore` also marks sensitive page regions (an MFA dialog), and
|
|
178
|
+
a target under one of those is plainly `covered`.
|
|
179
|
+
- **Everything else sees beneath it.** `list_targets` / `listTargets()` and
|
|
180
|
+
`listTabs()` report targets under the panel as `visible` and `inViewport`
|
|
181
|
+
(they never hit-test), `readPageText` reads the page text under it (never
|
|
182
|
+
the panel's own), resolution by id or text finds targets under it, and
|
|
183
|
+
`select_tab` clicks a tab under it.
|
|
184
|
+
- **Wholly hidden**: a target counts as `covered_by_agent` only when it
|
|
185
|
+
would be at least partly visible without the agent's UI; under the chat
|
|
186
|
+
window _and_ a sticky header, it is `covered`. `highlight`, `point` and
|
|
187
|
+
`scroll_to` then answer `hidden` with that reason.
|
|
188
|
+
- **Partly hidden**: when the agent's UI covers part of the target (a dock
|
|
189
|
+
over the right 40% of a heading cuts it; a window over a corner is drawn
|
|
190
|
+
over), the call still answers `ok`, with the effect cut at the panel, and
|
|
191
|
+
`output.partlyCoveredByAgent: true` when that is still so at the end.
|
|
192
|
+
- `clearTheWay(request)` is called at most once per `highlight`, `point`,
|
|
193
|
+
`scroll_to` or `select_tab`, whenever the agent's UI hides all or part of
|
|
194
|
+
the target, never for other covers. `request` is `{ tool, target: { id?,
|
|
195
|
+
label }, rect, cover: { element, rect }, viewport }`, all in viewport px:
|
|
196
|
+
`rect` is what the user would see of the target without the agent's UI
|
|
197
|
+
(`visibleWithoutAgent`), `cover.element` the outermost `agentUiSelector`
|
|
198
|
+
element hit and `cover.rect` its box over the target. Return (or resolve
|
|
199
|
+
to) `true` when you moved something; the call then waits for the target
|
|
200
|
+
and that element to stop moving (at most 1 s) and measures once more
|
|
201
|
+
(`select_tab` clicks straight away). No answer within
|
|
202
|
+
`CLEAR_THE_WAY_TIMEOUT_MS` (1000), a throw or a rejection count as "did
|
|
203
|
+
nothing". Results after a move carry `output.clearedTheWay: true`.
|
|
204
|
+
- **Keep room for the caption.** An outline's caption sits up to 34 px above
|
|
205
|
+
(or below) `rect`. `findClearPosition`'s default margin, `CLEAR_MARGIN`
|
|
206
|
+
(40), allows for it; use the same margin with `overlaps`. A ghost
|
|
207
|
+
cursor's label can reach further right (up to 240 px); pass a larger
|
|
208
|
+
margin for `point` if your window sits right of the target.
|
|
209
|
+
- Effects already on screen that later end up under the agent's UI (the
|
|
210
|
+
user drags the window over an outline) are hidden or cut as before; the
|
|
211
|
+
hook is not called for them.
|
|
212
|
+
- `findClearPosition(viewport, windowRect, keepClear, { margin, edge })`
|
|
213
|
+
returns the nearest top-left position for a floating window that keeps
|
|
214
|
+
`margin` px (default 40) from `keepClear` and `edge` px (default 8) from
|
|
215
|
+
the viewport edges (the current one if it already does), or null when
|
|
216
|
+
none fits. `overlaps(a, b, margin)` answers "is `a` in the way of `b`
|
|
217
|
+
plus `margin`?" for a dock at the other edge.
|
|
106
218
|
|
|
107
219
|
## Collapsed tab rows
|
|
108
220
|
|
|
@@ -141,8 +253,8 @@ attribute, and nothing but `role="tab"` is ever clicked.
|
|
|
141
253
|
- `ok` is reported only once the effect is in place (the effect is in the
|
|
142
254
|
store and its target is not covered, the tab is selected, the host's
|
|
143
255
|
navigation resolved).
|
|
144
|
-
- `[data-agent-ignore]`, the host `ignoreSelector` and
|
|
145
|
-
(`.chpa`) are invisible to resolution, snapshots and `readPageText`, which
|
|
256
|
+
- `[data-agent-ignore]`, the host `ignoreSelector` and `agentUiSelector`,
|
|
257
|
+
and the layer's own UI (`.chpa`) are invisible to resolution, snapshots and `readPageText`, which
|
|
146
258
|
also never reads password, hidden, card or one-time-code field values.
|
|
147
259
|
Treat page text as untrusted data on the server.
|
|
148
260
|
|
|
@@ -158,7 +270,8 @@ target) and a live region (`.chpa-sr`). Override `--chpa-accent` (default
|
|
|
158
270
|
`#6d28d9`) and `--chpa-z` (default `2147482000`, below the Access
|
|
159
271
|
Inspector). The layer stays on top so targets inside dialogs can be pointed
|
|
160
272
|
at; covers are handled by drawing only the visible part (an outline cut by
|
|
161
|
-
a header carries `data-cut="top"
|
|
273
|
+
a header carries `data-cut="top"`, one cut by a menu over its right end
|
|
274
|
+
`data-cut="right"`, and is left open on that side). Dark mode follows `.dark` / `[data-theme='dark']`; reduced
|
|
162
275
|
motion drops the glide and pulse.
|
|
163
276
|
|
|
164
277
|
## Testing
|
|
@@ -167,4 +280,6 @@ jsdom has no layout, so pass `configurePageAgent({ measure })` to supply
|
|
|
167
280
|
element boxes. The style rules (`hidden`, `display: none`, `visibility`,
|
|
168
281
|
`opacity: 0`) still run for real. jsdom has no hit testing either: nothing
|
|
169
282
|
counts as covered unless you pass `configurePageAgent({ hitTest: (x, y) =>
|
|
170
|
-
element | null })`.
|
|
283
|
+
element | null })`. A custom `hitTest` cannot look through the agent's UI,
|
|
284
|
+
so points it reports under `agentUiSelector` count as uncovered when
|
|
285
|
+
telling `covered_by_agent` from `covered`.
|
package/dist/index.d.ts
CHANGED
|
@@ -107,6 +107,13 @@ type PageCall = {
|
|
|
107
107
|
};
|
|
108
108
|
type PageToolName = PageCall['tool'];
|
|
109
109
|
type PageCallStatus = 'ok' | 'not_found' | 'ambiguous' | 'hidden' | 'refused' | 'error';
|
|
110
|
+
/**
|
|
111
|
+
* `output.reason` of a `hidden` result for a covered target: behind a modal
|
|
112
|
+
* dialog, under another part of the page, or under the agent's own UI (see
|
|
113
|
+
* {@link PageAgentConfig.agentUiSelector}). A closed set: agent servers
|
|
114
|
+
* mirror it as an enum.
|
|
115
|
+
*/
|
|
116
|
+
type CoveredReason = 'behind_dialog' | 'covered' | 'covered_by_agent';
|
|
110
117
|
/** What goes back to the model. `ok` only once the effect is confirmed. */
|
|
111
118
|
interface PageCallResult {
|
|
112
119
|
status: PageCallStatus;
|
|
@@ -161,6 +168,51 @@ type MeasureElement = (el: Element) => DOMRect | null;
|
|
|
161
168
|
* unusual hosts (canvas overlays, iframes) can swap it.
|
|
162
169
|
*/
|
|
163
170
|
type HitTestPoint = (x: number, y: number) => Element | null;
|
|
171
|
+
/** A box in viewport px (same shape as the geometry helpers' `Box`). */
|
|
172
|
+
interface ViewportBox {
|
|
173
|
+
top: number;
|
|
174
|
+
left: number;
|
|
175
|
+
width: number;
|
|
176
|
+
height: number;
|
|
177
|
+
}
|
|
178
|
+
/** What {@link ClearTheWayHandler} gets: enough to move the agent's window off a target. */
|
|
179
|
+
interface ClearTheWayRequest {
|
|
180
|
+
/** The tool whose target is covered (for `select_tab`: the tab, or its drop-down). */
|
|
181
|
+
tool: 'highlight' | 'point' | 'scroll_to' | 'select_tab';
|
|
182
|
+
/** The target, as the result would name it. */
|
|
183
|
+
target: {
|
|
184
|
+
id?: string;
|
|
185
|
+
label: string;
|
|
186
|
+
};
|
|
187
|
+
/**
|
|
188
|
+
* The part of the target the user would see without the agent's UI in
|
|
189
|
+
* the way (`visibleWithoutAgent`), in viewport px. Move the window off
|
|
190
|
+
* this, keeping room for the effect's caption: an outline's caption sits
|
|
191
|
+
* up to 34 px above or below it, which `findClearPosition`'s default
|
|
192
|
+
* margin (`CLEAR_MARGIN`, 40) allows for.
|
|
193
|
+
*/
|
|
194
|
+
rect: ViewportBox;
|
|
195
|
+
/** The agent's UI over the target. */
|
|
196
|
+
cover: {
|
|
197
|
+
/** The outermost element matching `agentUiSelector` that was hit, when known. */
|
|
198
|
+
element: Element | null;
|
|
199
|
+
/** Box of the agent's UI over the target, in viewport px. */
|
|
200
|
+
rect: ViewportBox;
|
|
201
|
+
};
|
|
202
|
+
viewport: {
|
|
203
|
+
width: number;
|
|
204
|
+
height: number;
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Host hook called when the agent's own UI covers all or part of a
|
|
209
|
+
* `highlight`, `point`, `scroll_to` or `select_tab` target (a docked panel
|
|
210
|
+
* over one end of a heading counts). Move (or collapse) that UI, then return
|
|
211
|
+
* (or resolve to) `true` if you did something; anything else means "did
|
|
212
|
+
* nothing". Called at most once per call, given {@link CLEAR_THE_WAY_TIMEOUT_MS}
|
|
213
|
+
* to answer; errors count as "did nothing".
|
|
214
|
+
*/
|
|
215
|
+
type ClearTheWayHandler = (request: ClearTheWayRequest) => boolean | void | Promise<boolean | void>;
|
|
164
216
|
interface PageAgentConfig {
|
|
165
217
|
/** Called by the `navigate` tool. Without it, `navigate` is refused. */
|
|
166
218
|
navigate?: NavigateHandler;
|
|
@@ -168,6 +220,16 @@ interface PageAgentConfig {
|
|
|
168
220
|
aliasAttributes?: string[];
|
|
169
221
|
/** Host CSS selector for regions the agent must never see or point at (in addition to `[data-agent-ignore]`). */
|
|
170
222
|
ignoreSelector?: string;
|
|
223
|
+
/**
|
|
224
|
+
* Host CSS selector for the agent's own UI: its floating chat window,
|
|
225
|
+
* dock and launcher. Never read or pointed at (like `ignoreSelector`),
|
|
226
|
+
* and a target hidden under it is reported as `covered_by_agent` instead
|
|
227
|
+
* of `covered`. Not `[data-agent-ignore]`: that also marks sensitive page
|
|
228
|
+
* regions (an MFA dialog), which are not the agent's UI.
|
|
229
|
+
*/
|
|
230
|
+
agentUiSelector?: string;
|
|
231
|
+
/** Called when a target is covered by the agent's own UI; see {@link ClearTheWayHandler}. */
|
|
232
|
+
clearTheWay?: ClearTheWayHandler;
|
|
171
233
|
/** Layout reader; see {@link MeasureElement}. */
|
|
172
234
|
measure?: MeasureElement;
|
|
173
235
|
/** Hit test used to tell whether a target is covered; see {@link HitTestPoint}. */
|
|
@@ -179,6 +241,8 @@ interface ResolvedPageAgentConfig {
|
|
|
179
241
|
navigate: NavigateHandler | null;
|
|
180
242
|
aliasAttributes: string[];
|
|
181
243
|
ignoreSelector: string | null;
|
|
244
|
+
agentUiSelector: string | null;
|
|
245
|
+
clearTheWay: ClearTheWayHandler | null;
|
|
182
246
|
measure: MeasureElement | null;
|
|
183
247
|
hitTest: HitTestPoint | null;
|
|
184
248
|
tabSettleMs: number;
|
|
@@ -265,7 +329,7 @@ declare function placeLabel(rect: RectLike, size: Size, viewport: Size, opts?: {
|
|
|
265
329
|
declare function pointerTarget(rect: RectLike, viewport: Size, margin?: number): Point;
|
|
266
330
|
/** Overlap of two boxes; null when they do not overlap. */
|
|
267
331
|
declare function intersectBoxes(a: RectLike, b: RectLike): Box | null;
|
|
268
|
-
/** Which sides of a target are cut off (by clipping, the viewport or a covering
|
|
332
|
+
/** Which sides of a target are cut off (by clipping, the viewport or a covering strip). */
|
|
269
333
|
interface CutSides {
|
|
270
334
|
top: boolean;
|
|
271
335
|
right: boolean;
|
|
@@ -289,14 +353,35 @@ interface TargetVisibility {
|
|
|
289
353
|
visible: Box | null;
|
|
290
354
|
/**
|
|
291
355
|
* The area the target is shown in: the viewport, narrowed by clipping
|
|
292
|
-
* ancestors and by
|
|
293
|
-
*
|
|
356
|
+
* ancestors and by the strips (sticky header, footer, side dock, a menu
|
|
357
|
+
* over one end) that cut it. Captions are kept inside it. Null when there
|
|
358
|
+
* is none.
|
|
294
359
|
*/
|
|
295
360
|
bounds: Box | null;
|
|
296
361
|
cut: CutSides;
|
|
297
|
-
/**
|
|
298
|
-
|
|
299
|
-
|
|
362
|
+
/**
|
|
363
|
+
* With `covered`: a modal dialog / inert page, another element on top, or
|
|
364
|
+
* the agent's own UI (see {@link VisibilityInput.coverAt}): the target
|
|
365
|
+
* would be at least partly visible if the agent's UI were not there.
|
|
366
|
+
*/
|
|
367
|
+
coveredBy?: 'modal' | 'element' | 'agent';
|
|
368
|
+
/**
|
|
369
|
+
* Set whenever the agent's own UI hides some of the target: all of it
|
|
370
|
+
* (`state: 'covered'`, `coveredBy: 'agent'`) or part of it (`visible` /
|
|
371
|
+
* `partial`: cut at a docked panel, or a window drawn over a corner).
|
|
372
|
+
* The box of the agent's UI over the target (the union of every agent-UI
|
|
373
|
+
* cover that overlaps it), in viewport px.
|
|
374
|
+
*/
|
|
375
|
+
agentCover?: Box;
|
|
376
|
+
/**
|
|
377
|
+
* Set with `agentCover`: the part of the target the user would see
|
|
378
|
+
* without the agent's UI in the way. Keep this clear to show all of it.
|
|
379
|
+
*/
|
|
380
|
+
visibleWithoutAgent?: Box;
|
|
381
|
+
/**
|
|
382
|
+
* Strips that cut the target (or a header-like cover over all of it), as
|
|
383
|
+
* measured: sticky headers and footers, docks, menus over one side.
|
|
384
|
+
*/
|
|
300
385
|
bands: Box[];
|
|
301
386
|
}
|
|
302
387
|
interface VisibilityInput {
|
|
@@ -309,27 +394,103 @@ interface VisibilityInput {
|
|
|
309
394
|
behindModal?: boolean;
|
|
310
395
|
/**
|
|
311
396
|
* Hit test: the box of whatever is drawn on top of the target at this
|
|
312
|
-
* viewport point, or null when the target itself (or nothing) is.
|
|
397
|
+
* viewport point, or null when the target itself (or nothing) is. Flag a
|
|
398
|
+
* cover that is the agent's own UI (its chat window, dock, launcher) with
|
|
399
|
+
* `agent: true`; a target covered only because of such covers is
|
|
400
|
+
* reported as `coveredBy: 'agent'`.
|
|
401
|
+
*/
|
|
402
|
+
coverAt?: (x: number, y: number) => CoverHit | null;
|
|
403
|
+
/**
|
|
404
|
+
* Hit test that looks through the agent's UI: what is on top of the
|
|
405
|
+
* target at this point once agent-UI covers are skipped. Used only to
|
|
406
|
+
* decide between `coveredBy: 'agent'` and `'element'`. Without it, a
|
|
407
|
+
* point under the agent's UI counts as uncovered there.
|
|
408
|
+
*/
|
|
409
|
+
coverBelowAgentAt?: (x: number, y: number) => CoverHit | null;
|
|
410
|
+
/**
|
|
411
|
+
* Sample grid per side (default 3, so 9 points), plus one probe just
|
|
412
|
+
* inside the middle of each edge of the visible part (see
|
|
413
|
+
* {@link computeVisibility}).
|
|
313
414
|
*/
|
|
314
|
-
coverAt?: (x: number, y: number) => RectLike | null;
|
|
315
|
-
/** Sample grid per side (default 3, so 9 points). */
|
|
316
415
|
grid?: number;
|
|
317
416
|
}
|
|
417
|
+
/** A cover box from a hit test; `agent` marks the agent's own UI. */
|
|
418
|
+
type CoverHit = RectLike & {
|
|
419
|
+
agent?: boolean;
|
|
420
|
+
};
|
|
318
421
|
/**
|
|
319
422
|
* Where the user can actually see a target: its box intersected with the
|
|
320
423
|
* viewport and every clipping ancestor, then hit-tested at a grid of points.
|
|
321
424
|
*
|
|
322
425
|
* Occlusion rules, chosen to be robust with coarse sampling:
|
|
323
426
|
* - something that covers the whole visible part (a dialog backdrop) hides it;
|
|
324
|
-
* -
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
427
|
+
* - a side strip, a cover that spans the visible part's full height and
|
|
428
|
+
* reaches past its left or right edge (a side dock, a menu over the end of
|
|
429
|
+
* a heading), or its full width and past its top or bottom edge (a sticky
|
|
430
|
+
* header or footer), cuts the target off exactly at the cover's edge, so
|
|
431
|
+
* the effect stops where the cover starts. The cover does not have to
|
|
432
|
+
* span the viewport, and cuts on several sides combine;
|
|
433
|
+
* - anything else on top (a floating button over a corner, a popover over
|
|
434
|
+
* the middle) hides the target only when it covers most of the grid
|
|
435
|
+
* points left uncut, and is otherwise drawn over: clipping around
|
|
436
|
+
* arbitrary shapes is not worth the wrong guesses.
|
|
437
|
+
*
|
|
438
|
+
* Sampling: a `grid` x `grid` set of points over the visible part (weighs
|
|
439
|
+
* covers for the "most of it" rule), plus a probe just inside the middle of
|
|
440
|
+
* each edge (finds side strips the grid falls short of, such as a menu over
|
|
441
|
+
* the last tenth of a wide heading: any strip of an edge's full length
|
|
442
|
+
* contains that edge's midpoint). After each cut the new edges are probed
|
|
443
|
+
* again, so a strip that only spans the part below a header is found too.
|
|
444
|
+
* Edge probes only find strips; they never count towards "most of it".
|
|
445
|
+
* 9 + 4 hit tests for an unobstructed target, 4 more per cut.
|
|
446
|
+
*
|
|
447
|
+
* Agent UI: when the target comes out `covered` and some covers were
|
|
448
|
+
* flagged `agent`, it is measured again looking through them
|
|
449
|
+
* (`coverBelowAgentAt`). If it would then be at least partly visible, the
|
|
450
|
+
* result is `coveredBy: 'agent'` with `agentCover` and
|
|
451
|
+
* `visibleWithoutAgent`; otherwise it stays `'element'` (something else
|
|
452
|
+
* covers it too, so moving the agent's UI would not help). A visible or
|
|
453
|
+
* partial target is drawn exactly as before (an agent panel over one side
|
|
454
|
+
* cuts it like any other strip), but when agent-UI covers overlap what it
|
|
455
|
+
* would show without them, it also gets `agentCover` and
|
|
456
|
+
* `visibleWithoutAgent`, so a host can move its UI off it.
|
|
331
457
|
*/
|
|
332
458
|
declare function computeVisibility(input: VisibilityInput): TargetVisibility;
|
|
459
|
+
/**
|
|
460
|
+
* Default gap {@link findClearPosition} keeps around the rect to keep
|
|
461
|
+
* clear: an outline's 4 px padding plus its 8 px gap and 22 px caption above
|
|
462
|
+
* or below the target, rounded up.
|
|
463
|
+
*/
|
|
464
|
+
declare const CLEAR_MARGIN = 40;
|
|
465
|
+
/**
|
|
466
|
+
* Whether two boxes overlap once `b` is grown by `margin` px on every side
|
|
467
|
+
* (touching edges do not count). For a docked panel: "would the target be
|
|
468
|
+
* clear if the panel were pinned to the other edge?" is
|
|
469
|
+
* `!overlaps(panelBoxAtOtherEdge, request.rect, CLEAR_MARGIN)`.
|
|
470
|
+
*/
|
|
471
|
+
declare function overlaps(a: RectLike, b: RectLike, margin?: number): boolean;
|
|
472
|
+
interface ClearPositionOptions {
|
|
473
|
+
/**
|
|
474
|
+
* Gap kept between the window and `keepClear`, in px. Default
|
|
475
|
+
* {@link CLEAR_MARGIN} (40): room for an outline's padding and its
|
|
476
|
+
* caption, drawn just above (or below) the target.
|
|
477
|
+
*/
|
|
478
|
+
margin?: number;
|
|
479
|
+
/** Gap kept between the window and the viewport edges, in px. Default 8. */
|
|
480
|
+
edge?: number;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Where to move a floating window (`windowRect`: the agent's chat window,
|
|
484
|
+
* say) so it no
|
|
485
|
+
* longer overlaps `keepClear` (the part of a target the user should see):
|
|
486
|
+
* the nearest top-left position, by straight-line distance from where the
|
|
487
|
+
* window is now, that keeps `margin` px between the two and `edge` px from
|
|
488
|
+
* the viewport edges. Returns the current position when the window already
|
|
489
|
+
* keeps clear (and fits; one that sticks out is moved back in), or null when no position fits (the window is too
|
|
490
|
+
* big for the room left beside, above and below `keepClear`). Pure; all
|
|
491
|
+
* boxes in viewport px.
|
|
492
|
+
*/
|
|
493
|
+
declare function findClearPosition(viewport: Size, windowRect: RectLike, keepClear: RectLike, opts?: ClearPositionOptions): Point | null;
|
|
333
494
|
|
|
334
495
|
declare const EFFECT_DEFAULT_MS = 8000;
|
|
335
496
|
declare const EFFECT_MIN_MS = 1000;
|
|
@@ -410,6 +571,8 @@ declare function nudgeIntoBounds(el: Element, visibility: TargetVisibility): boo
|
|
|
410
571
|
* control characters and malformed escapes.
|
|
411
572
|
*/
|
|
412
573
|
declare function parseAppPath(path: unknown): string | null;
|
|
574
|
+
/** How long `clearTheWay` has to answer before it counts as "did nothing". */
|
|
575
|
+
declare const CLEAR_THE_WAY_TIMEOUT_MS = 1000;
|
|
413
576
|
/**
|
|
414
577
|
* Run one tool call from the model. Accepts `unknown` on purpose: anything
|
|
415
578
|
* that is not `{ tool, input }` with a known tool and a valid input comes
|
|
@@ -581,7 +744,7 @@ declare function isBehindModal(el: Element): boolean;
|
|
|
581
744
|
* docks) hit-tested. See {@link computeVisibility} for the rules. Style
|
|
582
745
|
* checks (`hidden`, `display: none`, …) are not repeated here; callers
|
|
583
746
|
* check {@link isRendered} first. Cheap enough per animation frame for a
|
|
584
|
-
* handful of effects (
|
|
747
|
+
* handful of effects (13 hit tests each, 4 more per cut side; reads only).
|
|
585
748
|
*/
|
|
586
749
|
declare function measureVisibility(el: Element): TargetVisibility;
|
|
587
750
|
|
|
@@ -596,4 +759,4 @@ declare function collapsedTabMenu(tab: Element): Element | null | undefined;
|
|
|
596
759
|
/** Back to a clean slate: default config, no effects. Registrations are kept (their owners destroy them). */
|
|
597
760
|
declare function resetPageAgent(): void;
|
|
598
761
|
|
|
599
|
-
export { type AgentTarget, type AgentTargetHandle, type AgentTargetKind, type Box, CALL_LABEL_MAX, CANDIDATES_MAX, type Candidate, type CutSides, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, type EffectKind, type EffectOutcome, type HighlightInput, type HighlightOptions, type HighlightStyle, type HitTestPoint, type JsonShape, LAYER_CLASS, LIST_MAX, type LabelPlacement, type ListTargetsInput, type ListTargetsOutput, type MeasureElement, NAVIGATE_PATH_MAX, type NavigateHandler, type NavigateInput, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, type PageAgentConfig, type PageCall, type PageCallResult, type PageCallStatus, type PageEffect, type PageToolDescription, type PageToolName, type Point, type PointInput, type PointOptions, type ReadPageTextOptions, type RenderCache, type ResolveFailure, type ResolveOptions, type ResolveResult, type ResolvedPageAgentConfig, type ScrollOutcome, type ScrollToInput, type SelectTabInput, type Size, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, type TabSummary, type TargetRef, type TargetSummary, type TargetVisibility, type ViewportSize, type VisibilityInput, type VisibilityState, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
|
762
|
+
export { type AgentTarget, type AgentTargetHandle, type AgentTargetKind, type Box, CALL_LABEL_MAX, CANDIDATES_MAX, CLEAR_MARGIN, CLEAR_THE_WAY_TIMEOUT_MS, type Candidate, type ClearPositionOptions, type ClearTheWayHandler, type ClearTheWayRequest, type CoverHit, type CoveredReason, type CutSides, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, type EffectKind, type EffectOutcome, type HighlightInput, type HighlightOptions, type HighlightStyle, type HitTestPoint, type JsonShape, LAYER_CLASS, LIST_MAX, type LabelPlacement, type ListTargetsInput, type ListTargetsOutput, type MeasureElement, NAVIGATE_PATH_MAX, type NavigateHandler, type NavigateInput, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, type PageAgentConfig, type PageCall, type PageCallResult, type PageCallStatus, type PageEffect, type PageToolDescription, type PageToolName, type Point, type PointInput, type PointOptions, type ReadPageTextOptions, type RenderCache, type ResolveFailure, type ResolveOptions, type ResolveResult, type ResolvedPageAgentConfig, type ScrollOutcome, type ScrollToInput, type SelectTabInput, type Size, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, type TabSummary, type TargetRef, type TargetSummary, type TargetVisibility, type ViewportBox, type ViewportSize, type VisibilityInput, type VisibilityState, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, findClearPosition, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, overlaps, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,8 @@ var defaults = () => ({
|
|
|
6
6
|
navigate: null,
|
|
7
7
|
aliasAttributes: [],
|
|
8
8
|
ignoreSelector: null,
|
|
9
|
+
agentUiSelector: null,
|
|
10
|
+
clearTheWay: null,
|
|
9
11
|
measure: null,
|
|
10
12
|
hitTest: null,
|
|
11
13
|
tabSettleMs: DEFAULT_TAB_SETTLE_MS
|
|
@@ -25,6 +27,11 @@ function configurePageAgent(next) {
|
|
|
25
27
|
if ("ignoreSelector" in next) {
|
|
26
28
|
merged.ignoreSelector = validSelector(next.ignoreSelector) ? next.ignoreSelector.trim() : null;
|
|
27
29
|
}
|
|
30
|
+
if ("agentUiSelector" in next) {
|
|
31
|
+
merged.agentUiSelector = validSelector(next.agentUiSelector) ? next.agentUiSelector.trim() : null;
|
|
32
|
+
}
|
|
33
|
+
if ("clearTheWay" in next)
|
|
34
|
+
merged.clearTheWay = typeof next.clearTheWay === "function" ? next.clearTheWay : null;
|
|
28
35
|
if ("measure" in next) merged.measure = typeof next.measure === "function" ? next.measure : null;
|
|
29
36
|
if ("hitTest" in next) merged.hitTest = typeof next.hitTest === "function" ? next.hitTest : null;
|
|
30
37
|
if (next.tabSettleMs !== void 0 && Number.isFinite(next.tabSettleMs)) {
|
|
@@ -56,15 +63,22 @@ var BUILT_IN_IGNORE = `[data-agent-ignore],.${LAYER_CLASS}`;
|
|
|
56
63
|
var hasDom = () => typeof document !== "undefined";
|
|
57
64
|
function isIgnored(el, extraSelector) {
|
|
58
65
|
if (el.closest(BUILT_IN_IGNORE)) return true;
|
|
59
|
-
const
|
|
60
|
-
return safeClosest(el,
|
|
66
|
+
const { ignoreSelector, agentUiSelector } = getPageAgentConfig();
|
|
67
|
+
return safeClosest(el, ignoreSelector) !== null || safeClosest(el, agentUiSelector) !== null || safeClosest(el, extraSelector ?? null) !== null;
|
|
68
|
+
}
|
|
69
|
+
function agentUiRoot(el) {
|
|
70
|
+
const selector = getPageAgentConfig().agentUiSelector;
|
|
71
|
+
let root = null;
|
|
72
|
+
for (let node = safeClosest(el, selector); node; node = safeClosest(node.parentElement, selector))
|
|
73
|
+
root = node;
|
|
74
|
+
return root;
|
|
61
75
|
}
|
|
62
76
|
function safeClosest(el, selector) {
|
|
63
|
-
if (!selector) return
|
|
77
|
+
if (!selector || !el) return null;
|
|
64
78
|
try {
|
|
65
|
-
return el.closest(selector)
|
|
79
|
+
return el.closest(selector);
|
|
66
80
|
} catch {
|
|
67
|
-
return
|
|
81
|
+
return null;
|
|
68
82
|
}
|
|
69
83
|
}
|
|
70
84
|
function defaultMeasure(el) {
|
|
@@ -211,6 +225,8 @@ function pointerTarget(rect, viewport, margin = 12) {
|
|
|
211
225
|
}
|
|
212
226
|
var EDGE = 1;
|
|
213
227
|
var MIN_VISIBLE = 4;
|
|
228
|
+
var PROBE_INSET = 2;
|
|
229
|
+
var MAX_TRIM_ROUNDS = 6;
|
|
214
230
|
var viewportBox = (viewport) => ({
|
|
215
231
|
top: 0,
|
|
216
232
|
left: 0,
|
|
@@ -231,6 +247,38 @@ function containsBox(outer, inner) {
|
|
|
231
247
|
var containsPoint = (box, x, y) => x >= box.left && x <= box.left + box.width && y >= box.top && y <= box.top + box.height;
|
|
232
248
|
var NO_CUT = { top: false, right: false, bottom: false, left: false };
|
|
233
249
|
function computeVisibility(input) {
|
|
250
|
+
const coverAt = input.coverAt;
|
|
251
|
+
if (!coverAt) return measureOnce(input, void 0);
|
|
252
|
+
const agentBoxes = [];
|
|
253
|
+
const first = measureOnce(input, (x, y) => {
|
|
254
|
+
const hit = coverAt(x, y);
|
|
255
|
+
if (hit?.agent) {
|
|
256
|
+
const box = toBox(hit);
|
|
257
|
+
if (containsPoint(box, x, y) && !agentBoxes.some((b) => sameBox(b, box)))
|
|
258
|
+
agentBoxes.push(box);
|
|
259
|
+
}
|
|
260
|
+
return hit;
|
|
261
|
+
});
|
|
262
|
+
const fullyCovered = first.state === "covered" && first.coveredBy === "element";
|
|
263
|
+
const onScreen2 = first.state === "visible" || first.state === "partial";
|
|
264
|
+
if (agentBoxes.length === 0 || !fullyCovered && !onScreen2) return first;
|
|
265
|
+
const below = input.coverBelowAgentAt ?? coverAt;
|
|
266
|
+
const without = measureOnce(input, (x, y) => {
|
|
267
|
+
const hit = below(x, y);
|
|
268
|
+
return hit && !hit.agent ? hit : null;
|
|
269
|
+
});
|
|
270
|
+
const clear = without.visible;
|
|
271
|
+
if (!clear) return first;
|
|
272
|
+
const over = agentBoxes.filter((b) => intersectBoxes(b, clear));
|
|
273
|
+
if (onScreen2 && over.length === 0) return first;
|
|
274
|
+
return {
|
|
275
|
+
...first,
|
|
276
|
+
...fullyCovered ? { coveredBy: "agent" } : {},
|
|
277
|
+
agentCover: unionRects(over.length ? over : agentBoxes),
|
|
278
|
+
visibleWithoutAgent: clear
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
function measureOnce(input, coverAt) {
|
|
234
282
|
if (!input.rect) {
|
|
235
283
|
return { state: "unrendered", rect: null, visible: null, bounds: null, cut: NO_CUT, bands: [] };
|
|
236
284
|
}
|
|
@@ -257,24 +305,37 @@ function computeVisibility(input) {
|
|
|
257
305
|
return result("clipped", null);
|
|
258
306
|
}
|
|
259
307
|
const bands = [];
|
|
260
|
-
if (
|
|
308
|
+
if (coverAt) {
|
|
261
309
|
const n = Math.min(Math.max(Math.round(input.grid ?? 3), 1), 5);
|
|
262
310
|
const samples = [];
|
|
263
311
|
for (let row = 0; row < n; row++) {
|
|
264
312
|
for (let col = 0; col < n; col++) {
|
|
265
313
|
const x = visible.left + visible.width * (col + 0.5) / n;
|
|
266
314
|
const y = visible.top + visible.height * (row + 0.5) / n;
|
|
267
|
-
const hit =
|
|
315
|
+
const hit = coverAt(x, y);
|
|
268
316
|
const cover = hit ? toBox(hit) : null;
|
|
269
317
|
samples.push({ x, y, cover: cover && containsPoint(cover, x, y) ? cover : null });
|
|
270
318
|
}
|
|
271
319
|
}
|
|
272
320
|
const covers = [];
|
|
273
|
-
|
|
274
|
-
if (
|
|
275
|
-
}
|
|
321
|
+
const addCover = (cover) => {
|
|
322
|
+
if (cover && !covers.some((c) => sameBox(c, cover))) covers.push(cover);
|
|
323
|
+
};
|
|
324
|
+
for (const s of samples) addCover(s.cover);
|
|
325
|
+
const probed = /* @__PURE__ */ new Set();
|
|
326
|
+
const probeEdges = (box) => {
|
|
327
|
+
for (const [x, y] of edgeProbes(box)) {
|
|
328
|
+
const key = `${Math.round(x * 2)},${Math.round(y * 2)}`;
|
|
329
|
+
if (probed.has(key)) continue;
|
|
330
|
+
probed.add(key);
|
|
331
|
+
const hit = coverAt(x, y);
|
|
332
|
+
const cover = hit ? toBox(hit) : null;
|
|
333
|
+
addCover(cover && containsPoint(cover, x, y) ? cover : null);
|
|
334
|
+
}
|
|
335
|
+
};
|
|
336
|
+
probeEdges(visible);
|
|
276
337
|
let trimmed = true;
|
|
277
|
-
|
|
338
|
+
for (let round = 0; trimmed && visible && round < MAX_TRIM_ROUNDS; round++) {
|
|
278
339
|
trimmed = false;
|
|
279
340
|
for (const cover of covers) {
|
|
280
341
|
if (bands.includes(cover) || !visible || !intersectBoxes(cover, visible)) continue;
|
|
@@ -293,6 +354,9 @@ function computeVisibility(input) {
|
|
|
293
354
|
bands.push(cover);
|
|
294
355
|
trimmed = true;
|
|
295
356
|
}
|
|
357
|
+
if (trimmed && visible && visible.width >= MIN_VISIBLE && visible.height >= MIN_VISIBLE) {
|
|
358
|
+
probeEdges(visible);
|
|
359
|
+
}
|
|
296
360
|
}
|
|
297
361
|
if (!visible || visible.width < MIN_VISIBLE || visible.height < MIN_VISIBLE) {
|
|
298
362
|
return result("covered", null, { coveredBy: "element", bands });
|
|
@@ -307,6 +371,18 @@ function computeVisibility(input) {
|
|
|
307
371
|
const whole = !cut.top && !cut.right && !cut.bottom && !cut.left;
|
|
308
372
|
return { state: whole ? "visible" : "partial", rect, visible, bounds, cut, bands };
|
|
309
373
|
}
|
|
374
|
+
function edgeProbes(box) {
|
|
375
|
+
const dx = Math.min(PROBE_INSET, box.width / 4);
|
|
376
|
+
const dy = Math.min(PROBE_INSET, box.height / 4);
|
|
377
|
+
const cx = box.left + box.width / 2;
|
|
378
|
+
const cy = box.top + box.height / 2;
|
|
379
|
+
return [
|
|
380
|
+
[cx, box.top + dy],
|
|
381
|
+
[box.left + box.width - dx, cy],
|
|
382
|
+
[cx, box.top + box.height - dy],
|
|
383
|
+
[box.left + dx, cy]
|
|
384
|
+
];
|
|
385
|
+
}
|
|
310
386
|
function sameBox(a, b) {
|
|
311
387
|
return a.top === b.top && a.left === b.left && a.width === b.width && a.height === b.height;
|
|
312
388
|
}
|
|
@@ -344,6 +420,41 @@ function trimBand(visible, cover) {
|
|
|
344
420
|
if (spansY && cRight >= vRight - EDGE) return trim("right", cover.left);
|
|
345
421
|
return null;
|
|
346
422
|
}
|
|
423
|
+
var CLEAR_MARGIN = 40;
|
|
424
|
+
function overlaps(a, b, margin = 0) {
|
|
425
|
+
return intersectBoxes(a, inflate(b, Math.max(0, margin))) !== null;
|
|
426
|
+
}
|
|
427
|
+
function findClearPosition(viewport, windowRect, keepClear, opts = {}) {
|
|
428
|
+
const margin = Math.max(0, opts.margin ?? CLEAR_MARGIN);
|
|
429
|
+
const edge = Math.max(0, opts.edge ?? 8);
|
|
430
|
+
const w = windowRect.width;
|
|
431
|
+
const h = windowRect.height;
|
|
432
|
+
const minX = edge;
|
|
433
|
+
const minY = edge;
|
|
434
|
+
const maxX = viewport.width - edge - w;
|
|
435
|
+
const maxY = viewport.height - edge - h;
|
|
436
|
+
if (maxX < minX || maxY < minY) return null;
|
|
437
|
+
const keep = inflate(keepClear, margin);
|
|
438
|
+
const clampX = (x) => Math.min(Math.max(x, minX), maxX);
|
|
439
|
+
const clampY = (y) => Math.min(Math.max(y, minY), maxY);
|
|
440
|
+
const overlaps2 = (left, top) => intersectBoxes({ left, top, width: w, height: h }, keep) !== null;
|
|
441
|
+
const here = { left: windowRect.left, top: windowRect.top };
|
|
442
|
+
const keepRight = keep.left + keep.width;
|
|
443
|
+
const keepBottom = keep.top + keep.height;
|
|
444
|
+
const candidates = [
|
|
445
|
+
{ left: clampX(here.left), top: clampY(here.top) },
|
|
446
|
+
{ left: keep.left - w, top: clampY(here.top) },
|
|
447
|
+
{ left: keepRight, top: clampY(here.top) },
|
|
448
|
+
{ left: clampX(here.left), top: keep.top - h },
|
|
449
|
+
{ left: clampX(here.left), top: keepBottom }
|
|
450
|
+
].filter(
|
|
451
|
+
(c) => c.left >= minX && c.left <= maxX && c.top >= minY && c.top <= maxY && !overlaps2(c.left, c.top)
|
|
452
|
+
);
|
|
453
|
+
if (candidates.length === 0) return null;
|
|
454
|
+
const distance = (c) => Math.hypot(c.left - here.left, c.top - here.top);
|
|
455
|
+
const best = candidates.reduce((a, b) => distance(b) < distance(a) ? b : a);
|
|
456
|
+
return { top: Math.round(best.top), left: Math.round(best.left) };
|
|
457
|
+
}
|
|
347
458
|
|
|
348
459
|
// src/registry.ts
|
|
349
460
|
var TARGET_LABEL_MAX = 80;
|
|
@@ -617,10 +728,12 @@ function quote(value) {
|
|
|
617
728
|
// src/visibility.ts
|
|
618
729
|
var CLIPS = /^(hidden|clip|auto|scroll|overlay)$/;
|
|
619
730
|
var COVER_WALK_MAX = 16;
|
|
620
|
-
function defaultHitTest(x, y) {
|
|
731
|
+
function defaultHitTest(x, y, skipAgentUi = false) {
|
|
621
732
|
if (typeof document.elementsFromPoint !== "function") return null;
|
|
622
733
|
for (const el of document.elementsFromPoint(x, y)) {
|
|
623
|
-
if (
|
|
734
|
+
if (el.closest(`.${LAYER_CLASS}`)) continue;
|
|
735
|
+
if (skipAgentUi && agentUiRoot(el)) continue;
|
|
736
|
+
return el;
|
|
624
737
|
}
|
|
625
738
|
return null;
|
|
626
739
|
}
|
|
@@ -632,6 +745,13 @@ function hitTest(x, y) {
|
|
|
632
745
|
return null;
|
|
633
746
|
}
|
|
634
747
|
}
|
|
748
|
+
function hitTestBelowAgentUi(x, y) {
|
|
749
|
+
try {
|
|
750
|
+
return defaultHitTest(x, y, true);
|
|
751
|
+
} catch {
|
|
752
|
+
return null;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
635
755
|
var toBox2 = (r) => ({ top: r.top, left: r.left, width: r.width, height: r.height });
|
|
636
756
|
function clippingBoxes(el) {
|
|
637
757
|
const view = el.ownerDocument.defaultView;
|
|
@@ -680,35 +800,56 @@ function isBehindModal(el) {
|
|
|
680
800
|
const top = modals[modals.length - 1];
|
|
681
801
|
return !!top && !top.contains(el);
|
|
682
802
|
}
|
|
683
|
-
function coverBox(el,
|
|
684
|
-
const hit = hitTest(x, y);
|
|
803
|
+
function coverBox(el, hit, onAgent) {
|
|
685
804
|
if (!hit || hit === el || el.contains(hit) || hit.contains(el)) return null;
|
|
686
805
|
let box = layoutRect(hit);
|
|
687
806
|
if (!box) return null;
|
|
688
|
-
|
|
807
|
+
const agent = agentUiRoot(hit);
|
|
808
|
+
let node = hit === agent ? null : hit.parentElement;
|
|
689
809
|
for (let i = 0; node && i < COVER_WALK_MAX && !node.contains(el); i++) {
|
|
690
810
|
const outer = layoutRect(node);
|
|
691
811
|
if (!outer || !encloses(outer, box)) break;
|
|
692
812
|
box = outer;
|
|
813
|
+
if (node === agent) break;
|
|
693
814
|
node = node.parentElement;
|
|
694
815
|
}
|
|
695
|
-
return box;
|
|
816
|
+
if (!agent) return box;
|
|
817
|
+
onAgent?.(agent);
|
|
818
|
+
return { top: box.top, left: box.left, width: box.width, height: box.height, agent: true };
|
|
696
819
|
}
|
|
697
820
|
function encloses(outer, inner) {
|
|
698
821
|
return outer.top <= inner.top + 1 && outer.left <= inner.left + 1 && outer.bottom >= inner.bottom - 1 && outer.right >= inner.right - 1;
|
|
699
822
|
}
|
|
700
823
|
function measureVisibility(el) {
|
|
824
|
+
return measureVisibilityDetail(el).visibility;
|
|
825
|
+
}
|
|
826
|
+
function measureVisibilityDetail(el) {
|
|
701
827
|
const rect = hasDom() && el.isConnected ? layoutRect(el) : null;
|
|
702
828
|
if (!rect) {
|
|
703
|
-
return
|
|
829
|
+
return {
|
|
830
|
+
visibility: computeVisibility({ rect: null, viewport: viewportSize() }),
|
|
831
|
+
agentElement: null
|
|
832
|
+
};
|
|
704
833
|
}
|
|
705
|
-
|
|
834
|
+
const agents = [];
|
|
835
|
+
const onAgent = (root) => {
|
|
836
|
+
if (!agents.includes(root)) agents.push(root);
|
|
837
|
+
};
|
|
838
|
+
const config2 = getPageAgentConfig();
|
|
839
|
+
const visibility = computeVisibility({
|
|
706
840
|
rect,
|
|
707
841
|
viewport: viewportSize(),
|
|
708
842
|
clips: clippingBoxes(el),
|
|
709
843
|
behindModal: isBehindModal(el),
|
|
710
|
-
coverAt: (x, y) => coverBox(el, x, y)
|
|
844
|
+
coverAt: (x, y) => coverBox(el, hitTest(x, y), onAgent),
|
|
845
|
+
// A custom hit test cannot look through the agent's UI: its agent hits
|
|
846
|
+
// then count as uncovered points (see computeVisibility).
|
|
847
|
+
...config2.agentUiSelector && !config2.hitTest ? { coverBelowAgentAt: (x, y) => coverBox(el, hitTestBelowAgentUi(x, y)) } : {}
|
|
711
848
|
});
|
|
849
|
+
return {
|
|
850
|
+
visibility,
|
|
851
|
+
agentElement: visibility.agentCover ? agents[0] ?? null : null
|
|
852
|
+
};
|
|
712
853
|
}
|
|
713
854
|
|
|
714
855
|
// src/controller.ts
|
|
@@ -1304,11 +1445,12 @@ var MENU_NOTE = "The tabs are shown as a drop-down menu here (the page is narrow
|
|
|
1304
1445
|
var wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1305
1446
|
var SCROLL_SETTLE_MAX_MS = 1e3;
|
|
1306
1447
|
var SCROLL_SETTLE_STEP_MS = 50;
|
|
1307
|
-
|
|
1308
|
-
|
|
1448
|
+
var CLEAR_THE_WAY_TIMEOUT_MS = 1e3;
|
|
1449
|
+
async function settleLayout(...els) {
|
|
1450
|
+
const key = () => els.map((el) => {
|
|
1309
1451
|
const r = layoutRect(el);
|
|
1310
1452
|
return r ? `${Math.round(r.top)},${Math.round(r.left)}` : "";
|
|
1311
|
-
};
|
|
1453
|
+
}).join(";");
|
|
1312
1454
|
let last = key();
|
|
1313
1455
|
let stable = 0;
|
|
1314
1456
|
for (let waited = 0; waited < SCROLL_SETTLE_MAX_MS; waited += SCROLL_SETTLE_STEP_MS) {
|
|
@@ -1325,25 +1467,78 @@ async function settleLayout(el) {
|
|
|
1325
1467
|
async function reveal(el, scrolledAlready) {
|
|
1326
1468
|
let scrolled = scrolledAlready ?? scrollIntoContainers(el);
|
|
1327
1469
|
if (scrolled) await settleLayout(el);
|
|
1328
|
-
let
|
|
1329
|
-
if (visibility.bands.length > 0 && nudgeIntoBounds(el, visibility)) {
|
|
1470
|
+
let measured = measureVisibilityDetail(el);
|
|
1471
|
+
if (measured.visibility.bands.length > 0 && nudgeIntoBounds(el, measured.visibility)) {
|
|
1330
1472
|
scrolled = true;
|
|
1331
1473
|
await settleLayout(el);
|
|
1332
|
-
|
|
1474
|
+
measured = measureVisibilityDetail(el);
|
|
1475
|
+
}
|
|
1476
|
+
return { scrolled, ...measured };
|
|
1477
|
+
}
|
|
1478
|
+
function within(promise, ms, fallback) {
|
|
1479
|
+
return new Promise((resolve) => {
|
|
1480
|
+
const timer = setTimeout(() => resolve(fallback), ms);
|
|
1481
|
+
promise.then(
|
|
1482
|
+
(value) => {
|
|
1483
|
+
clearTimeout(timer);
|
|
1484
|
+
resolve(value);
|
|
1485
|
+
},
|
|
1486
|
+
() => {
|
|
1487
|
+
clearTimeout(timer);
|
|
1488
|
+
resolve(fallback);
|
|
1489
|
+
}
|
|
1490
|
+
);
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1493
|
+
async function askToClearTheWay(tool, resolved, visibility, agentElement) {
|
|
1494
|
+
const handler = getPageAgentConfig().clearTheWay;
|
|
1495
|
+
const { agentCover, visibleWithoutAgent } = visibility;
|
|
1496
|
+
if (!handler || !agentCover || !visibleWithoutAgent) return false;
|
|
1497
|
+
const request = {
|
|
1498
|
+
tool,
|
|
1499
|
+
target: summary(resolved),
|
|
1500
|
+
rect: { ...visibleWithoutAgent },
|
|
1501
|
+
cover: { element: agentElement, rect: { ...agentCover } },
|
|
1502
|
+
viewport: viewportSize()
|
|
1503
|
+
};
|
|
1504
|
+
let answer;
|
|
1505
|
+
try {
|
|
1506
|
+
answer = handler(request);
|
|
1507
|
+
} catch {
|
|
1508
|
+
return false;
|
|
1333
1509
|
}
|
|
1334
|
-
return
|
|
1510
|
+
return await within(Promise.resolve(answer), CLEAR_THE_WAY_TIMEOUT_MS, false) === true;
|
|
1511
|
+
}
|
|
1512
|
+
async function revealForCall(tool, resolved, scrolledAlready) {
|
|
1513
|
+
const { scrolled, visibility, agentElement } = await reveal(resolved.el, scrolledAlready);
|
|
1514
|
+
if (!visibility.agentCover) return { scrolled, visibility, clearedTheWay: false };
|
|
1515
|
+
if (!await askToClearTheWay(tool, resolved, visibility, agentElement)) {
|
|
1516
|
+
return { scrolled, visibility, clearedTheWay: false };
|
|
1517
|
+
}
|
|
1518
|
+
await settleLayout(...agentElement ? [resolved.el, agentElement] : [resolved.el]);
|
|
1519
|
+
return {
|
|
1520
|
+
scrolled,
|
|
1521
|
+
visibility: measureVisibilityDetail(resolved.el).visibility,
|
|
1522
|
+
clearedTheWay: true
|
|
1523
|
+
};
|
|
1335
1524
|
}
|
|
1336
1525
|
var onScreen = (v) => v.state === "visible" || v.state === "partial";
|
|
1526
|
+
var COVER_REASON = {
|
|
1527
|
+
modal: "behind_dialog",
|
|
1528
|
+
element: "covered",
|
|
1529
|
+
agent: "covered_by_agent"
|
|
1530
|
+
};
|
|
1337
1531
|
function coveredResult(visibility, label2) {
|
|
1338
1532
|
if (visibility.state !== "covered") return null;
|
|
1339
1533
|
const what = `"${label2.replace(/"/g, "'")}"`;
|
|
1340
|
-
const
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
message: modal ? `${what} is behind a dialog right now, so it cannot be shown. Point at something in the dialog, or ask the user to close it.` : `${what} is covered by another part of the page (such as a header, panel or popup), so it cannot be shown right now.`,
|
|
1344
|
-
output: { reason: modal ? "behind_dialog" : "covered" }
|
|
1345
|
-
};
|
|
1534
|
+
const reason = COVER_REASON[visibility.coveredBy ?? "element"];
|
|
1535
|
+
const message = reason === "behind_dialog" ? `${what} is behind a dialog right now, so it cannot be shown. Point at something in the dialog, or ask the user to close it.` : reason === "covered_by_agent" ? `${what} is covered by the assistant's own window, so it cannot be shown right now. Ask the user to move or minimise the assistant window, then try again.` : `${what} is covered by another part of the page (such as a header, panel or popup), so it cannot be shown right now.`;
|
|
1536
|
+
return { status: "hidden", message, output: { reason } };
|
|
1346
1537
|
}
|
|
1538
|
+
var agentNotes = (clearedTheWay, visibility) => ({
|
|
1539
|
+
...clearedTheWay ? { clearedTheWay: true } : {},
|
|
1540
|
+
...visibility.agentCover && onScreen(visibility) ? { partlyCoveredByAgent: true } : {}
|
|
1541
|
+
});
|
|
1347
1542
|
async function confirmSelected(el, settleMs) {
|
|
1348
1543
|
const selected = () => el.isConnected && el.getAttribute("aria-selected") === "true";
|
|
1349
1544
|
await Promise.resolve();
|
|
@@ -1390,7 +1585,7 @@ var handlers = {
|
|
|
1390
1585
|
const durationMs2 = optionalDuration(input.durationMs);
|
|
1391
1586
|
const resolved = resolveTarget(target2);
|
|
1392
1587
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1393
|
-
const { scrolled, visibility } = await
|
|
1588
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall("highlight", resolved);
|
|
1394
1589
|
const covered = coveredResult(visibility, resolved.label);
|
|
1395
1590
|
if (covered) return covered;
|
|
1396
1591
|
if (!isRendered(resolved.el)) {
|
|
@@ -1410,6 +1605,7 @@ var handlers = {
|
|
|
1410
1605
|
style,
|
|
1411
1606
|
scrolled,
|
|
1412
1607
|
inViewport: onScreen(visibility),
|
|
1608
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1413
1609
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1414
1610
|
}
|
|
1415
1611
|
};
|
|
@@ -1421,7 +1617,7 @@ var handlers = {
|
|
|
1421
1617
|
const durationMs2 = optionalDuration(input.durationMs);
|
|
1422
1618
|
const resolved = resolveTarget(target2);
|
|
1423
1619
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1424
|
-
const { scrolled, visibility } = await
|
|
1620
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall("point", resolved);
|
|
1425
1621
|
const covered = coveredResult(visibility, resolved.label);
|
|
1426
1622
|
if (covered) return covered;
|
|
1427
1623
|
if (!isRendered(resolved.el)) {
|
|
@@ -1440,6 +1636,7 @@ var handlers = {
|
|
|
1440
1636
|
target: summary(resolved),
|
|
1441
1637
|
scrolled,
|
|
1442
1638
|
inViewport: onScreen(visibility),
|
|
1639
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1443
1640
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1444
1641
|
}
|
|
1445
1642
|
};
|
|
@@ -1451,13 +1648,19 @@ var handlers = {
|
|
|
1451
1648
|
if (resolved.status !== "ok") return failure(resolved);
|
|
1452
1649
|
const outcome = scrollToTarget(resolved.el);
|
|
1453
1650
|
if (outcome.status !== "ok") return failure(outcome);
|
|
1454
|
-
const { scrolled, visibility } = await
|
|
1651
|
+
const { scrolled, visibility, clearedTheWay } = await revealForCall(
|
|
1652
|
+
"scroll_to",
|
|
1653
|
+
resolved,
|
|
1654
|
+
outcome.scrolled
|
|
1655
|
+
);
|
|
1656
|
+
if (visibility.coveredBy === "agent") return coveredResult(visibility, resolved.label);
|
|
1455
1657
|
return {
|
|
1456
1658
|
status: "ok",
|
|
1457
1659
|
output: {
|
|
1458
1660
|
target: summary(resolved),
|
|
1459
1661
|
scrolled,
|
|
1460
1662
|
inViewport: onScreen(visibility),
|
|
1663
|
+
...agentNotes(clearedTheWay, visibility),
|
|
1461
1664
|
...visibility.state === "covered" ? { covered: true } : {},
|
|
1462
1665
|
...resolved.shownAs ? { shownAs: resolved.shownAs } : {}
|
|
1463
1666
|
}
|
|
@@ -1496,6 +1699,8 @@ var handlers = {
|
|
|
1496
1699
|
if (isBehindModal(shown)) {
|
|
1497
1700
|
return { status: "hidden", message: "That tab is behind a dialog right now." };
|
|
1498
1701
|
}
|
|
1702
|
+
const measured = measureVisibilityDetail(shown);
|
|
1703
|
+
const clearedTheWay = !!measured.visibility.agentCover && await askToClearTheWay("select_tab", resolved, measured.visibility, measured.agentElement);
|
|
1499
1704
|
el.click();
|
|
1500
1705
|
if (!await confirmSelected(el, getPageAgentConfig().tabSettleMs)) {
|
|
1501
1706
|
return { status: "error", message: "The tab did not become selected." };
|
|
@@ -1503,7 +1708,12 @@ var handlers = {
|
|
|
1503
1708
|
return {
|
|
1504
1709
|
status: "ok",
|
|
1505
1710
|
...asMenu ? { message: `${MENU_NOTE} The menu now shows this tab.` } : {},
|
|
1506
|
-
output: {
|
|
1711
|
+
output: {
|
|
1712
|
+
target: summary(resolved),
|
|
1713
|
+
alreadySelected: false,
|
|
1714
|
+
...extra,
|
|
1715
|
+
...clearedTheWay ? { clearedTheWay: true } : {}
|
|
1716
|
+
}
|
|
1507
1717
|
};
|
|
1508
1718
|
},
|
|
1509
1719
|
async navigate(raw) {
|
|
@@ -1544,4 +1754,4 @@ function resetPageAgent() {
|
|
|
1544
1754
|
resetPageAgentConfig();
|
|
1545
1755
|
}
|
|
1546
1756
|
|
|
1547
|
-
export { CALL_LABEL_MAX, CANDIDATES_MAX, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, LAYER_CLASS, LIST_MAX, NAVIGATE_PATH_MAX, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
|
1757
|
+
export { CALL_LABEL_MAX, CANDIDATES_MAX, CLEAR_MARGIN, CLEAR_THE_WAY_TIMEOUT_MS, EFFECTS_MAX, EFFECT_DEFAULT_MS, EFFECT_LABEL_MAX, EFFECT_MAX_MS, EFFECT_MIN_MS, LAYER_CLASS, LIST_MAX, NAVIGATE_PATH_MAX, PAGE_TEXT_DEFAULT, PAGE_TEXT_MAX, PAGE_TOOLS, PAGE_TOOL_NAMES, TABLIST_MENU_ATTR, TARGET_ID_MAX, TARGET_LABEL_MAX, TARGET_TEXT_MAX, TRUNCATION_MARKER, attachPageAgent, clampDuration, clampToViewport, clearEffects, clippingBoxes, collapsedTabMenu, computeVisibility, configurePageAgent, executePageCall, findClearPosition, getAgentTargets, getEffects, getPageAgentConfig, highlight, inflate, intersectBoxes, intersectsViewport, isBehindModal, isFullyInViewport, isRendered, listTabs, listTargets, measureVisibility, notifyNavigated, nudgeIntoBounds, onAgentTargetsChange, onEffectsChange, overlaps, parseAppPath, placeLabel, point, pointerTarget, prefersReducedMotion, readPageText, rectOf, registerAgentTarget, resetPageAgent, resolveTarget, scanAgentAttributes, scrollIntoContainers, scrollToTarget, targetIdOf, unionRects, viewportSize, visibilityOf };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@classic-homes/page-agent-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Framework-agnostic page layer for an in-app AI agent: named targets, safe target resolution, a page snapshot, highlight / spotlight / pointer effects and one schema-validated executor for model tool calls",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "dist/index.js",
|