@nexface/tools 0.1.15-alpha.1 → 0.1.15-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/actions.js +34 -10
- package/dist/browser/controller.js +53 -74
- package/dist/browser/diagnostics.js +8 -3
- package/dist/browser/dom.js +12 -28
- package/dist/browser/feedback.js +6 -5
- package/dist/browser/focus.d.ts +0 -1
- package/dist/browser/focus.js +1 -13
- package/dist/browser/locator.js +3 -9
- package/dist/browser/prompt.d.ts +1 -1
- package/dist/browser/prompt.generated.d.ts +1 -1
- package/dist/browser/prompt.generated.js +6 -3
- package/dist/browser/prompt.js +0 -1
- package/dist/browser/snapshot.js +141 -120
- package/dist/browser/toolset.js +42 -36
- package/dist/browser/types.d.ts +1 -1
- package/dist/browser/types.js +0 -1
- package/dist/browser/worker-runner.js +0 -1
- package/dist/definition.d.ts +1 -3
- package/dist/definition.js +2 -2
- package/dist/index.browser.js +0 -1
- package/dist/index.js +0 -1
- package/dist/internal-react.d.ts +1 -1
- package/dist/internal-react.js +3 -4
- package/dist/internal-types.js +0 -1
- package/dist/navigation.js +0 -1
- package/dist/operation-lock.d.ts +1 -1
- package/dist/operation-lock.js +36 -19
- package/dist/operation-runtime.js +41 -13
- package/dist/operation-surface.js +118 -41
- package/dist/operation-types.js +0 -1
- package/dist/operation-view.d.ts +1 -1
- package/dist/operation-view.js +232 -82
- package/dist/react.js +9 -14
- package/dist/registration.js +4 -3
- package/dist/runtime.d.ts +4 -2
- package/dist/runtime.js +114 -52
- package/dist/shared/definition.js +2 -3
- package/dist/shared/errors.d.ts +1 -1
- package/dist/shared/errors.js +0 -1
- package/dist/shared/page-protocol.js +0 -1
- package/dist/shared/page-tools.js +2 -3
- package/dist/shared/runtime-controller.d.ts +1 -0
- package/dist/shared/runtime-controller.js +0 -1
- package/dist/shared/runtime-execution.js +0 -1
- package/dist/shared/schema.js +45 -19
- package/dist/styles.css +882 -109
- package/dist/tool-search.d.ts +11 -0
- package/dist/tool-search.js +117 -0
- package/dist/types.js +0 -1
- package/dist/webmcp.js +39 -13
- package/package.json +8 -4
- package/dist/browser/actions.js.map +0 -1
- package/dist/browser/controller.js.map +0 -1
- package/dist/browser/diagnostics.js.map +0 -1
- package/dist/browser/dom.js.map +0 -1
- package/dist/browser/feedback.js.map +0 -1
- package/dist/browser/focus.js.map +0 -1
- package/dist/browser/locator.js.map +0 -1
- package/dist/browser/prompt.generated.js.map +0 -1
- package/dist/browser/prompt.js.map +0 -1
- package/dist/browser/snapshot.js.map +0 -1
- package/dist/browser/toolset.js.map +0 -1
- package/dist/browser/types.js.map +0 -1
- package/dist/browser/worker-runner.js.map +0 -1
- package/dist/definition.js.map +0 -1
- package/dist/index.browser.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/internal-react.js.map +0 -1
- package/dist/internal-types.js.map +0 -1
- package/dist/navigation.js.map +0 -1
- package/dist/operation-lock.js.map +0 -1
- package/dist/operation-runtime.js.map +0 -1
- package/dist/operation-surface.js.map +0 -1
- package/dist/operation-types.js.map +0 -1
- package/dist/operation-view.js.map +0 -1
- package/dist/react.js.map +0 -1
- package/dist/registration.js.map +0 -1
- package/dist/runtime.js.map +0 -1
- package/dist/shared/definition.js.map +0 -1
- package/dist/shared/errors.js.map +0 -1
- package/dist/shared/page-protocol.js.map +0 -1
- package/dist/shared/page-tools.js.map +0 -1
- package/dist/shared/runtime-controller.js.map +0 -1
- package/dist/shared/runtime-execution.js.map +0 -1
- package/dist/shared/schema.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/webmcp.js.map +0 -1
|
@@ -4,7 +4,10 @@ export const BROWSER_TOOLS_INSTRUCTIONS = `Page automation is a fallback for pag
|
|
|
4
4
|
Tool selection:
|
|
5
5
|
- Prefer currently registered developer-provided application tools whenever they match the requested capability.
|
|
6
6
|
- The current page is available through nxf_getSnapshot, even when application tools are empty. Read it before asking the user for page information.
|
|
7
|
-
-
|
|
7
|
+
- When the current page lacks needed information or actions, follow available navigation tools or observed page links and inspect the destination's tools or content. Perform these steps yourself rather than asking the user to open the page.
|
|
8
|
+
- If no application tool covers the remaining work, observe the page with nxf_getSnapshot and continue through page automation. Missing application tools alone are not a blocker.
|
|
9
|
+
- Continue until tool results or page observations confirm completion or reveal a blocker. If blocked, state what is needed to continue.
|
|
10
|
+
- When an application tool reports a permission restriction, required user confirmation, or a blocked write, follow its recovery requirements. Do not bypass the restriction through page automation.
|
|
8
11
|
- Call nxf_getSnapshot only when page information is needed.
|
|
9
12
|
- Call nxf_executeAction only after a current snapshot exists and the page must be operated.
|
|
10
13
|
- When calling a tool, keep assistant content concise. Resolve uncertain page state with a fresh snapshot.
|
|
@@ -30,7 +33,7 @@ nxf_executeAction rules:
|
|
|
30
33
|
- selectOption(value) works on native <select> elements. Pass an option's value string, or an array of value strings for a multiple select; option labels are not accepted as values.
|
|
31
34
|
- press(key) dispatches keydown and keyup events only; it does not perform native text entry or guarantee form submission. Use fill() to enter text and click() on the submit control to submit.
|
|
32
35
|
- On semantic Locators, waitFor({ state?, timeout? }) accepts "visible" (the default), "hidden", or "attached"; timeout is a non-negative number of milliseconds, capped by the per-action limit. On a Snapshot ref, waitFor() only checks that the ref is still valid.
|
|
33
|
-
- text() returns
|
|
36
|
+
- text() requires one unique element and returns its current value for input, textarea, and select elements; for other elements it returns normalized text content. exists() only checks for a match; it does not establish a unique, enabled, or unobstructed write target.
|
|
34
37
|
- Example with a ref from the current Snapshot: await ui.ref("@12").fill("Lin"); const value = await ui.ref("@12").text();
|
|
35
38
|
- Example for a control that appears during execution: await ui.getByRole("dialog", { name: "Edit" }).getByRole("button", { name: "Save" }).waitFor({ state: "visible" });
|
|
36
39
|
|
|
@@ -38,6 +41,7 @@ Targeting policy:
|
|
|
38
41
|
- For a write target already present in the Snapshot, prefer ui.ref().
|
|
39
42
|
- Runtime Locators are for strict unique semantic lookup, including elements that appear after execution begins.
|
|
40
43
|
- Scope repeated controls through a dialog, form, row, list item, or named region.
|
|
44
|
+
- If a control's input node is covered by its own observed label or trigger, operate that label or trigger and inspect the resulting state. Pointer interception alone is not a permission restriction.
|
|
41
45
|
- Prefer stable business identifiers. Never guess by position or invent a ref.
|
|
42
46
|
- Ambiguous writes are rejected. If the target is ambiguous or unknown, stop and observe again.
|
|
43
47
|
|
|
@@ -54,4 +58,3 @@ Execution policy:
|
|
|
54
58
|
- Return JSON-serializable data when reading information from the page.
|
|
55
59
|
- A successful Page action receipt proves only that the action ran. Confirm business success through visible feedback, field readback, or an application tool result.
|
|
56
60
|
- nxf_executeAction automatically returns a best-effort viewport snapshot. If refsExpired, postSnapshotTimedOut, or snapshotUnavailable is true, or the expected page state is absent, call nxf_getSnapshot before another write or nxf_finish.`;
|
|
57
|
-
//# sourceMappingURL=prompt.generated.js.map
|
package/dist/browser/prompt.js
CHANGED
package/dist/browser/snapshot.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { collectElementActions, collectPageActions } from "./actions.js";
|
|
2
|
-
import { composedChildren, composedParent, computeNameInfo, computeRole, computeState, createIdentity, createScrollableIdentity, findNearestHeadingText, getElementScrollability, isActionableRole, isContextRole, isElementInViewport, isElementRendered, normalizeText,
|
|
2
|
+
import { composedChildren, composedParent, computeNameInfo, computeRole, computeState, createIdentity, createScrollableIdentity, findNearestHeadingText, getElementScrollability, isActionableRole, isContextRole, isElementInViewport, isElementRendered, normalizeText, } from "./dom.js";
|
|
3
3
|
const DEFAULT_LIMITS = {
|
|
4
4
|
viewportMaxChars: 18_000,
|
|
5
5
|
viewportMaxVisitedNodes: 12_000,
|
|
@@ -27,15 +27,6 @@ const STRUCTURAL_ROLES = new Set([
|
|
|
27
27
|
"tree",
|
|
28
28
|
"listitem",
|
|
29
29
|
]);
|
|
30
|
-
const PORTAL_ROLES = new Set([
|
|
31
|
-
"alert",
|
|
32
|
-
"listbox",
|
|
33
|
-
"menu",
|
|
34
|
-
"progressbar",
|
|
35
|
-
"status",
|
|
36
|
-
"tooltip",
|
|
37
|
-
"tree",
|
|
38
|
-
]);
|
|
39
30
|
const REDUNDANT_WRAPPER_ROLES = new Set(["group", "listitem"]);
|
|
40
31
|
function checkExecution(context) {
|
|
41
32
|
if (!context)
|
|
@@ -104,21 +95,19 @@ function isTextInScope(node, context) {
|
|
|
104
95
|
const style = cachedStyle(parent, context.styles);
|
|
105
96
|
if (!isElementRendered(parent, style))
|
|
106
97
|
return false;
|
|
107
|
-
if (context.scope === "full")
|
|
108
|
-
return true;
|
|
109
|
-
const parentRect = cachedRect(parent, context.rects);
|
|
110
|
-
if (!isElementInViewport(parent, parentRect))
|
|
111
|
-
return false;
|
|
112
98
|
const range = document.createRange();
|
|
113
99
|
range.selectNodeContents(node);
|
|
114
100
|
const rect = range.getBoundingClientRect();
|
|
115
101
|
range.detach();
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
102
|
+
const inViewport = isElementInViewport(parent, rect);
|
|
103
|
+
if (context.scope !== "full" && !inViewport)
|
|
104
|
+
return false;
|
|
105
|
+
if (!inViewport)
|
|
106
|
+
return context.scope === "full";
|
|
107
|
+
// Pointer-transparent text can be visible even when another element receives input.
|
|
108
|
+
if (style.pointerEvents === "none")
|
|
109
|
+
return true;
|
|
110
|
+
return sampleRect(parent, rect, context, false) !== false;
|
|
122
111
|
}
|
|
123
112
|
function isElementInScope(element, context) {
|
|
124
113
|
const style = cachedStyle(element, context.styles);
|
|
@@ -130,36 +119,109 @@ function isElementInScope(element, context) {
|
|
|
130
119
|
}
|
|
131
120
|
return isElementInViewport(element, cachedRect(element, context.rects));
|
|
132
121
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
let current =
|
|
137
|
-
|
|
138
|
-
if (current === ancestor)
|
|
139
|
-
return true;
|
|
140
|
-
current = composedParent(current);
|
|
122
|
+
// Only opaque backgrounds provide evidence of visual occlusion. Pointer interception alone does not.
|
|
123
|
+
function hasOpaqueCover(hit, target, context, x, y) {
|
|
124
|
+
const ancestors = new Set();
|
|
125
|
+
for (let current = target; current; current = composedParent(current)) {
|
|
126
|
+
ancestors.add(current);
|
|
141
127
|
}
|
|
142
|
-
|
|
128
|
+
const labels = target instanceof HTMLInputElement ||
|
|
129
|
+
target instanceof HTMLTextAreaElement ||
|
|
130
|
+
target instanceof HTMLSelectElement ||
|
|
131
|
+
target instanceof HTMLButtonElement
|
|
132
|
+
? new Set(target.labels ?? [])
|
|
133
|
+
: undefined;
|
|
134
|
+
let opaque = false;
|
|
135
|
+
for (let current = hit; current; current = composedParent(current)) {
|
|
136
|
+
// A native label still exposes its associated control, even with an opaque background.
|
|
137
|
+
if (labels?.has(current))
|
|
138
|
+
return false;
|
|
139
|
+
if (ancestors.has(current))
|
|
140
|
+
break;
|
|
141
|
+
const style = cachedStyle(current, context.styles);
|
|
142
|
+
if (Number(style.opacity || 1) < 1)
|
|
143
|
+
return false;
|
|
144
|
+
const color = style.backgroundColor;
|
|
145
|
+
if (/^rgb\([^/]+\)$/.test(color) || /^rgba\(.+,\s*1(?:\.0+)?\)$/.test(color)) {
|
|
146
|
+
const rect = cachedRect(current, context.rects);
|
|
147
|
+
if (x >= rect.left && x < rect.right && y >= rect.top && y < rect.bottom)
|
|
148
|
+
opaque = true;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
return opaque;
|
|
143
152
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
// Hit testing is bounded per snapshot and never scans a candidate's descendants.
|
|
154
|
+
function sampleRect(element, rect, context, allowDescendantHit = true) {
|
|
155
|
+
const left = Math.max(0, rect.left);
|
|
156
|
+
const top = Math.max(0, rect.top);
|
|
157
|
+
const right = Math.min(window.innerWidth, rect.right);
|
|
158
|
+
const bottom = Math.min(window.innerHeight, rect.bottom);
|
|
159
|
+
if (right <= left || bottom <= top)
|
|
160
|
+
return undefined;
|
|
161
|
+
if (typeof document.elementFromPoint !== "function")
|
|
162
|
+
return undefined;
|
|
163
|
+
let uncertain = false;
|
|
164
|
+
for (const [x, y] of [
|
|
165
|
+
[0.5, 0.5],
|
|
166
|
+
[0.2, 0.2],
|
|
167
|
+
[0.8, 0.2],
|
|
168
|
+
[0.2, 0.8],
|
|
169
|
+
[0.8, 0.8],
|
|
170
|
+
]) {
|
|
171
|
+
if (context.hitTests >= 4096) {
|
|
172
|
+
context.truncated = true;
|
|
173
|
+
return undefined;
|
|
174
|
+
}
|
|
175
|
+
context.hitTests += 1;
|
|
176
|
+
checkExecution(context.execution);
|
|
177
|
+
let hit = document.elementFromPoint(left + (right - left) * x, top + (bottom - top) * y);
|
|
178
|
+
while (hit?.shadowRoot) {
|
|
179
|
+
if (context.hitTests >= 4096) {
|
|
180
|
+
context.truncated = true;
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
context.hitTests += 1;
|
|
184
|
+
checkExecution(context.execution);
|
|
185
|
+
const inner = hit.shadowRoot.elementFromPoint?.(left + (right - left) * x, top + (bottom - top) * y);
|
|
186
|
+
if (!inner || inner === hit)
|
|
187
|
+
break;
|
|
188
|
+
hit = inner;
|
|
189
|
+
}
|
|
190
|
+
for (let current = hit; current; current = composedParent(current)) {
|
|
191
|
+
if (current === element)
|
|
192
|
+
return true;
|
|
193
|
+
if (!allowDescendantHit)
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
if (!allowDescendantHit &&
|
|
197
|
+
!hasOpaqueCover(hit, element, context, left + (right - left) * x, top + (bottom - top) * y))
|
|
198
|
+
uncertain = true;
|
|
152
199
|
}
|
|
153
|
-
return false;
|
|
200
|
+
return uncertain ? undefined : false;
|
|
201
|
+
}
|
|
202
|
+
function elementHit(element, context) {
|
|
203
|
+
if (context.hits.has(element))
|
|
204
|
+
return context.hits.get(element);
|
|
205
|
+
const clientRects = Array.from(element.getClientRects()).filter((rect) => rect.width > 0 && rect.height > 0);
|
|
206
|
+
const rects = clientRects.length ? clientRects : [cachedRect(element, context.rects)];
|
|
207
|
+
let result;
|
|
208
|
+
for (const rect of rects.filter((rect) => isElementInViewport(element, rect)).slice(0, 2)) {
|
|
209
|
+
const hit = sampleRect(element, rect, context);
|
|
210
|
+
if (hit === true) {
|
|
211
|
+
result = true;
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
if (hit === false)
|
|
215
|
+
result = false;
|
|
216
|
+
}
|
|
217
|
+
context.hits.set(element, result);
|
|
218
|
+
return result;
|
|
154
219
|
}
|
|
155
220
|
function normalizeNodeChildren(node) {
|
|
156
221
|
const merged = [];
|
|
157
222
|
for (const child of node.children) {
|
|
158
223
|
const previous = merged.at(-1);
|
|
159
|
-
if (child.role === "text" &&
|
|
160
|
-
previous?.role === "text" &&
|
|
161
|
-
!child.element &&
|
|
162
|
-
!previous.element) {
|
|
224
|
+
if (child.role === "text" && previous?.role === "text" && !child.element && !previous.element) {
|
|
163
225
|
previous.name = normalizeText(`${previous.name} ${child.name}`);
|
|
164
226
|
previous.normalizedName = previous.name;
|
|
165
227
|
continue;
|
|
@@ -167,14 +229,12 @@ function normalizeNodeChildren(node) {
|
|
|
167
229
|
merged.push(child);
|
|
168
230
|
}
|
|
169
231
|
const directText = normalizeText(merged
|
|
170
|
-
.filter((child) => child.role === "text")
|
|
232
|
+
.filter((child) => child.role === "text" && !child.element)
|
|
171
233
|
.map((child) => child.name)
|
|
172
234
|
.join(" "));
|
|
173
235
|
node.children =
|
|
174
|
-
node.normalizedName &&
|
|
175
|
-
|
|
176
|
-
node.normalizedName === directText
|
|
177
|
-
? merged.filter((child) => child.role !== "text")
|
|
236
|
+
node.normalizedName && directText && node.normalizedName === directText
|
|
237
|
+
? merged.filter((child) => child.role !== "text" || child.element)
|
|
178
238
|
: merged;
|
|
179
239
|
const onlyChild = node.children.length === 1 ? node.children[0] : undefined;
|
|
180
240
|
if (REDUNDANT_WRAPPER_ROLES.has(node.role) &&
|
|
@@ -199,10 +259,8 @@ function visitNode(node, context) {
|
|
|
199
259
|
if (node instanceof Text) {
|
|
200
260
|
if (!isTextInScope(node, context))
|
|
201
261
|
return [];
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
return [];
|
|
205
|
-
return [{
|
|
262
|
+
return [
|
|
263
|
+
{
|
|
206
264
|
role: "text",
|
|
207
265
|
name: normalizeText(node.data),
|
|
208
266
|
normalizedName: normalizeText(node.data),
|
|
@@ -210,7 +268,8 @@ function visitNode(node, context) {
|
|
|
210
268
|
state: {},
|
|
211
269
|
actionable: false,
|
|
212
270
|
children: [],
|
|
213
|
-
}
|
|
271
|
+
},
|
|
272
|
+
];
|
|
214
273
|
}
|
|
215
274
|
if (!(node instanceof Element))
|
|
216
275
|
return [];
|
|
@@ -223,7 +282,7 @@ function visitNode(node, context) {
|
|
|
223
282
|
return [];
|
|
224
283
|
const children = composedChildren(node).flatMap((child) => visitNode(child, context));
|
|
225
284
|
const inScope = isElementInScope(node, context);
|
|
226
|
-
const
|
|
285
|
+
const hit = inScope ? elementHit(node, context) : undefined;
|
|
227
286
|
const role = computeRole(node);
|
|
228
287
|
const state = computeState(node);
|
|
229
288
|
const inferredActionable = isActionableRole(role);
|
|
@@ -233,25 +292,39 @@ function visitNode(node, context) {
|
|
|
233
292
|
node.onclick === null &&
|
|
234
293
|
!node.hasAttribute("onclick") &&
|
|
235
294
|
!node.isContentEditable;
|
|
236
|
-
const scrollable = inScope &&
|
|
295
|
+
const scrollable = inScope &&
|
|
296
|
+
(hit === true || typeof document.elementFromPoint !== "function") &&
|
|
297
|
+
(!inferredActionable || focusOnlyActionable)
|
|
237
298
|
? getElementScrollability(node, style)
|
|
238
299
|
: undefined;
|
|
239
|
-
const actionable = inferredActionable &&
|
|
300
|
+
const actionable = inferredActionable &&
|
|
301
|
+
!scrollable &&
|
|
302
|
+
(hit === true || typeof document.elementFromPoint !== "function");
|
|
240
303
|
const hasState = Object.keys(state).length > 0;
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
304
|
+
// Pointer interception can prevent an action without hiding a control's semantics.
|
|
305
|
+
const observedControl = inScope &&
|
|
306
|
+
inferredActionable &&
|
|
307
|
+
hit === false &&
|
|
308
|
+
sampleRect(node, cachedRect(node, context.rects), context, false) !== false;
|
|
309
|
+
const keep = observedControl ||
|
|
310
|
+
(hit !== false &&
|
|
311
|
+
((inScope && (actionable || scrollable || STRUCTURAL_ROLES.has(role) || hasState)) ||
|
|
312
|
+
(children.length > 0 && (isContextRole(role) || role === "table" || role === "list"))));
|
|
244
313
|
if (!keep)
|
|
245
314
|
return children;
|
|
246
315
|
const computedName = computeNameInfo(node);
|
|
316
|
+
const visibleName = computedName.source === "content"
|
|
317
|
+
? children
|
|
318
|
+
.map((child) => child.name)
|
|
319
|
+
.filter(Boolean)
|
|
320
|
+
.join(" ")
|
|
321
|
+
: computedName.value;
|
|
247
322
|
const scrollableName = scrollable && computedName.source === "content"
|
|
248
323
|
? findNearestHeadingText(node)
|
|
249
|
-
:
|
|
324
|
+
: visibleName || (scrollable ? findNearestHeadingText(node) : "");
|
|
250
325
|
const renderNode = {
|
|
251
326
|
element: node,
|
|
252
|
-
role: scrollable && (!role || role === "actionable")
|
|
253
|
-
? "scrollable"
|
|
254
|
-
: role || "group",
|
|
327
|
+
role: scrollable && (!role || role === "actionable") ? "scrollable" : role || "text",
|
|
255
328
|
name: scrollableName,
|
|
256
329
|
normalizedName: normalizeText(scrollableName),
|
|
257
330
|
nameSource: computedName.source,
|
|
@@ -263,50 +336,6 @@ function visitNode(node, context) {
|
|
|
263
336
|
normalizeNodeChildren(renderNode);
|
|
264
337
|
return [renderNode];
|
|
265
338
|
}
|
|
266
|
-
function isVisibleModal(element) {
|
|
267
|
-
const role = computeRole(element);
|
|
268
|
-
const ariaModal = role === "dialog" && element.getAttribute("aria-modal") === "true";
|
|
269
|
-
const alertDialog = role === "alertdialog";
|
|
270
|
-
let nativeModal = false;
|
|
271
|
-
if (element instanceof HTMLDialogElement && element.open) {
|
|
272
|
-
try {
|
|
273
|
-
nativeModal = element.matches(":modal");
|
|
274
|
-
}
|
|
275
|
-
catch {
|
|
276
|
-
nativeModal = element.open;
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
return (ariaModal || alertDialog || nativeModal) && isElementInViewport(element);
|
|
280
|
-
}
|
|
281
|
-
function findActiveModal(execution) {
|
|
282
|
-
let activeModal;
|
|
283
|
-
let visited = 0;
|
|
284
|
-
for (const element of walkComposedElements()) {
|
|
285
|
-
visited += 1;
|
|
286
|
-
if (visited % 128 === 0)
|
|
287
|
-
checkExecution(execution);
|
|
288
|
-
if (isVisibleModal(element))
|
|
289
|
-
activeModal = element;
|
|
290
|
-
}
|
|
291
|
-
return activeModal;
|
|
292
|
-
}
|
|
293
|
-
function findPortalRoots(activeModal, execution) {
|
|
294
|
-
const roots = new Set();
|
|
295
|
-
let visited = 0;
|
|
296
|
-
for (const element of walkComposedElements()) {
|
|
297
|
-
visited += 1;
|
|
298
|
-
if (visited % 128 === 0)
|
|
299
|
-
checkExecution(execution);
|
|
300
|
-
if (isDescendantOrSelf(element, activeModal))
|
|
301
|
-
continue;
|
|
302
|
-
const role = computeRole(element);
|
|
303
|
-
if ((PORTAL_ROLES.has(role) || element.hasAttribute("aria-live")) &&
|
|
304
|
-
isElementInViewport(element)) {
|
|
305
|
-
roots.add(element);
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
return roots;
|
|
309
|
-
}
|
|
310
339
|
function serializeState(state) {
|
|
311
340
|
const entries = Object.entries(state);
|
|
312
341
|
if (entries.length === 0)
|
|
@@ -340,13 +369,11 @@ function renderTree(nodes, context) {
|
|
|
340
369
|
const name = node.name ? ` ${JSON.stringify(normalizeText(node.name))}` : "";
|
|
341
370
|
const state = serializeState(node.state);
|
|
342
371
|
const scrollability = serializeScrollability(node.scrollable);
|
|
343
|
-
const nextRef = node.actionable || node.scrollable
|
|
344
|
-
? `@${context.refs.size + 1}`
|
|
345
|
-
: undefined;
|
|
372
|
+
const nextRef = node.actionable || node.scrollable ? `@${context.refs.size + 1}` : undefined;
|
|
346
373
|
const refText = nextRef ? ` [ref=${nextRef}]` : "";
|
|
347
374
|
const prefix = node.role === "text" ? `${indent}- text:` : `${indent}- ${node.role}`;
|
|
348
375
|
const line = node.role === "text"
|
|
349
|
-
? `${prefix} ${JSON.stringify(normalizeText(node.name))}`
|
|
376
|
+
? `${prefix} ${JSON.stringify(normalizeText(node.name))}${state}`
|
|
350
377
|
: `${prefix}${name}${state}${scrollability}${refText}`;
|
|
351
378
|
const added = line.length + (context.lines.length > 0 ? 1 : 0);
|
|
352
379
|
if (context.usedChars + added > context.maxChars) {
|
|
@@ -376,13 +403,10 @@ export async function createSnapshot(scope = "viewport", limits = {}, execution,
|
|
|
376
403
|
await nextAnimationFrame(execution);
|
|
377
404
|
checkExecution(execution);
|
|
378
405
|
const resolvedLimits = { ...DEFAULT_LIMITS, ...limits };
|
|
379
|
-
const activeModal = findActiveModal(execution);
|
|
380
406
|
const walkContext = {
|
|
381
407
|
scope,
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
? findPortalRoots(activeModal, execution)
|
|
385
|
-
: new Set(),
|
|
408
|
+
hits: new WeakMap(),
|
|
409
|
+
hitTests: 0,
|
|
386
410
|
styles: new WeakMap(),
|
|
387
411
|
rects: new WeakMap(),
|
|
388
412
|
visitedNodes: 0,
|
|
@@ -402,9 +426,7 @@ export async function createSnapshot(scope = "viewport", limits = {}, execution,
|
|
|
402
426
|
const renderContext = {
|
|
403
427
|
...(actions ? { actions } : {}),
|
|
404
428
|
refs: registry.refs,
|
|
405
|
-
maxChars: scope === "viewport"
|
|
406
|
-
? resolvedLimits.viewportMaxChars
|
|
407
|
-
: resolvedLimits.fullMaxChars,
|
|
429
|
+
maxChars: scope === "viewport" ? resolvedLimits.viewportMaxChars : resolvedLimits.fullMaxChars,
|
|
408
430
|
usedChars: 0,
|
|
409
431
|
lines: [],
|
|
410
432
|
truncated: false,
|
|
@@ -429,4 +451,3 @@ export async function createSnapshot(scope = "viewport", limits = {}, execution,
|
|
|
429
451
|
export async function getSnapshot(scope = "viewport", limits = {}) {
|
|
430
452
|
return (await createSnapshot(scope, limits)).snapshot;
|
|
431
453
|
}
|
|
432
|
-
//# sourceMappingURL=snapshot.js.map
|
package/dist/browser/toolset.js
CHANGED
|
@@ -27,11 +27,7 @@ const DEFAULT_SNAPSHOT_LIMITS = {
|
|
|
27
27
|
fullMaxVisitedNodes: 40_000,
|
|
28
28
|
};
|
|
29
29
|
function isIgnoredMutationNode(node) {
|
|
30
|
-
const element = node instanceof Element
|
|
31
|
-
? node
|
|
32
|
-
: node instanceof Text
|
|
33
|
-
? node.parentElement
|
|
34
|
-
: null;
|
|
30
|
+
const element = node instanceof Element ? node : node instanceof Text ? node.parentElement : null;
|
|
35
31
|
return Boolean(element?.closest("[data-nexface-ignore]"));
|
|
36
32
|
}
|
|
37
33
|
function isRelevantMutation(record) {
|
|
@@ -40,8 +36,7 @@ function isRelevantMutation(record) {
|
|
|
40
36
|
if (record.type !== "childList")
|
|
41
37
|
return true;
|
|
42
38
|
const changedNodes = [...record.addedNodes, ...record.removedNodes];
|
|
43
|
-
return
|
|
44
|
-
|| changedNodes.some((node) => !isIgnoredMutationNode(node)));
|
|
39
|
+
return changedNodes.length === 0 || changedNodes.some((node) => !isIgnoredMutationNode(node));
|
|
45
40
|
}
|
|
46
41
|
let activeHistoryPatch;
|
|
47
42
|
function subscribeHistoryChanges(subscriber) {
|
|
@@ -223,12 +218,12 @@ function validateInput(definition, input) {
|
|
|
223
218
|
}
|
|
224
219
|
}
|
|
225
220
|
export function browserToolsAvailable() {
|
|
226
|
-
return (typeof window !== "undefined"
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
221
|
+
return (typeof window !== "undefined" &&
|
|
222
|
+
typeof document !== "undefined" &&
|
|
223
|
+
typeof Worker !== "undefined" &&
|
|
224
|
+
typeof Blob !== "undefined" &&
|
|
225
|
+
typeof URL?.createObjectURL === "function" &&
|
|
226
|
+
typeof MutationObserver !== "undefined");
|
|
232
227
|
}
|
|
233
228
|
export function createBrowserToolset(options = {}) {
|
|
234
229
|
if (!browserToolsAvailable()) {
|
|
@@ -236,19 +231,14 @@ export function createBrowserToolset(options = {}) {
|
|
|
236
231
|
}
|
|
237
232
|
const limits = {
|
|
238
233
|
...DEFAULT_EXECUTION_LIMITS,
|
|
239
|
-
maxDurationMs: options.limits?.maxDurationMs
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
?? DEFAULT_EXECUTION_LIMITS.maxResultChars,
|
|
243
|
-
singleActionTimeoutMs: options.limits?.singleActionTimeoutMs
|
|
244
|
-
?? DEFAULT_EXECUTION_LIMITS.singleActionTimeoutMs,
|
|
234
|
+
maxDurationMs: options.limits?.maxDurationMs ?? DEFAULT_EXECUTION_LIMITS.maxDurationMs,
|
|
235
|
+
maxResultChars: options.limits?.maxResultChars ?? DEFAULT_EXECUTION_LIMITS.maxResultChars,
|
|
236
|
+
singleActionTimeoutMs: options.limits?.singleActionTimeoutMs ?? DEFAULT_EXECUTION_LIMITS.singleActionTimeoutMs,
|
|
245
237
|
};
|
|
246
238
|
const snapshotLimits = {
|
|
247
239
|
...DEFAULT_SNAPSHOT_LIMITS,
|
|
248
|
-
viewportMaxChars: options.snapshotLimits?.viewportMaxChars
|
|
249
|
-
|
|
250
|
-
fullMaxChars: options.snapshotLimits?.fullMaxChars
|
|
251
|
-
?? DEFAULT_SNAPSHOT_LIMITS.fullMaxChars,
|
|
240
|
+
viewportMaxChars: options.snapshotLimits?.viewportMaxChars ?? DEFAULT_SNAPSHOT_LIMITS.viewportMaxChars,
|
|
241
|
+
fullMaxChars: options.snapshotLimits?.fullMaxChars ?? DEFAULT_SNAPSHOT_LIMITS.fullMaxChars,
|
|
252
242
|
};
|
|
253
243
|
const feedback = new ActionFeedback();
|
|
254
244
|
const controller = new PageController(limits, feedback);
|
|
@@ -285,7 +275,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
285
275
|
const scope = normalizedInput.scope;
|
|
286
276
|
const deadline = performance.now() + limits.maxDurationMs;
|
|
287
277
|
signal.throwIfAborted();
|
|
288
|
-
return (await observe(scope, { deadline, signal }, pageDecision ? "actions" : "text"))
|
|
278
|
+
return (await observe(scope, { deadline, signal }, pageDecision ? "actions" : "text"))
|
|
279
|
+
.snapshot;
|
|
289
280
|
},
|
|
290
281
|
};
|
|
291
282
|
const executeObserved = async (signal, execute, cancel, mode = "code", onFeedback, assertActive) => {
|
|
@@ -316,7 +307,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
316
307
|
}
|
|
317
308
|
result = await measurePagePhase("execution", () => execute(activeRegistry, workerBudget));
|
|
318
309
|
signal.throwIfAborted();
|
|
319
|
-
if (result.error &&
|
|
310
|
+
if (result.error &&
|
|
311
|
+
["EXECUTION_TIMEOUT", "WORKER_ERROR", "EXECUTION_OUTCOME_UNKNOWN"].includes(result.error.code)) {
|
|
320
312
|
result.error.code = "EXECUTION_OUTCOME_UNKNOWN";
|
|
321
313
|
}
|
|
322
314
|
if (!result.ok && result.completedActions.length === 0) {
|
|
@@ -325,9 +317,7 @@ export function createBrowserToolset(options = {}) {
|
|
|
325
317
|
try {
|
|
326
318
|
const settled = await measurePagePhase("settle", () => tracker.waitForSettle({ deadline, signal }));
|
|
327
319
|
signal.throwIfAborted();
|
|
328
|
-
if (settled.timedOut
|
|
329
|
-
&& settled.urlChanged
|
|
330
|
-
&& !settled.mutationObservedAfterUrlChange) {
|
|
320
|
+
if (settled.timedOut && settled.urlChanged && !settled.mutationObservedAfterUrlChange) {
|
|
331
321
|
result.snapshotUnavailable = true;
|
|
332
322
|
return expireRefs(result);
|
|
333
323
|
}
|
|
@@ -382,7 +372,9 @@ export function createBrowserToolset(options = {}) {
|
|
|
382
372
|
if (!bundle || bundle.snapshot.snapshotId !== input.snapshotId)
|
|
383
373
|
return undefined;
|
|
384
374
|
const selected = bundle.actions?.get(input.actionId);
|
|
385
|
-
if (!selected ||
|
|
375
|
+
if (!selected ||
|
|
376
|
+
selected.command.method === "scrollBy" ||
|
|
377
|
+
selected.command.method === "waitForTimeout")
|
|
386
378
|
return undefined;
|
|
387
379
|
const target = selected.command.target;
|
|
388
380
|
const element = target?.type === "ref" ? bundle.registry.refs.get(target.ref)?.element : undefined;
|
|
@@ -396,14 +388,21 @@ export function createBrowserToolset(options = {}) {
|
|
|
396
388
|
const selected = bundle.actions.get(input.actionId);
|
|
397
389
|
if (!selected)
|
|
398
390
|
throw new AgentToolError("INVALID_INPUT", "Action is not a candidate of this observation.");
|
|
399
|
-
if (selected.command.method === "fill"
|
|
391
|
+
if (selected.command.method === "fill"
|
|
392
|
+
? typeof input.text !== "string" || !input.text.trim()
|
|
393
|
+
: input.text !== undefined) {
|
|
400
394
|
throw new AgentToolError("INVALID_INPUT", "Only fill actions require a non-empty text value.");
|
|
401
395
|
}
|
|
402
396
|
if (selected.command.method === "selectOption") {
|
|
403
397
|
const target = selected.command.target;
|
|
404
398
|
const element = target.type === "ref" ? bundle.registry.refs.get(target.ref)?.element : undefined;
|
|
405
|
-
const options = element instanceof HTMLSelectElement
|
|
406
|
-
|
|
399
|
+
const options = element instanceof HTMLSelectElement
|
|
400
|
+
? [...element.options].filter((option) => option.value === selected.command.args[0])
|
|
401
|
+
: [];
|
|
402
|
+
if (options.length !== 1 ||
|
|
403
|
+
options[0].disabled ||
|
|
404
|
+
options[0].closest("optgroup[disabled]") ||
|
|
405
|
+
options[0].label.slice(0, 500) !== selected.candidate.option) {
|
|
407
406
|
throw new AgentToolError("INVALID_INPUT", "The observed option changed or is no longer available. Observe again.");
|
|
408
407
|
}
|
|
409
408
|
}
|
|
@@ -413,7 +412,9 @@ export function createBrowserToolset(options = {}) {
|
|
|
413
412
|
let timer;
|
|
414
413
|
try {
|
|
415
414
|
const command = {
|
|
416
|
-
...selected.command,
|
|
415
|
+
...selected.command,
|
|
416
|
+
executionId,
|
|
417
|
+
commandSeq: 1,
|
|
417
418
|
args: selected.command.method === "fill" ? [input.text] : selected.command.args,
|
|
418
419
|
};
|
|
419
420
|
const response = await Promise.race([
|
|
@@ -421,7 +422,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
421
422
|
if (selected.optionElement) {
|
|
422
423
|
const target = selected.command.target;
|
|
423
424
|
const element = target.type === "ref" ? registry.refs.get(target.ref)?.element : undefined;
|
|
424
|
-
if (!selected.optionElement.isConnected ||
|
|
425
|
+
if (!selected.optionElement.isConnected ||
|
|
426
|
+
selected.optionElement.closest("select") !== element) {
|
|
425
427
|
throw new AgentToolError("INVALID_INPUT", "The observed option was replaced. Observe again.");
|
|
426
428
|
}
|
|
427
429
|
}
|
|
@@ -434,7 +436,12 @@ export function createBrowserToolset(options = {}) {
|
|
|
434
436
|
}),
|
|
435
437
|
]);
|
|
436
438
|
const receipt = controller.completeExecution(executionId);
|
|
437
|
-
return {
|
|
439
|
+
return {
|
|
440
|
+
executionId,
|
|
441
|
+
...receipt,
|
|
442
|
+
ok: response.ok,
|
|
443
|
+
...(response.error ? { error: response.error } : {}),
|
|
444
|
+
};
|
|
438
445
|
}
|
|
439
446
|
finally {
|
|
440
447
|
clearTimeout(timer);
|
|
@@ -449,4 +456,3 @@ export function createBrowserToolset(options = {}) {
|
|
|
449
456
|
handlers: new Map(toolHandlers.map((tool) => [tool.definition.name, tool])),
|
|
450
457
|
};
|
|
451
458
|
}
|
|
452
|
-
//# sourceMappingURL=toolset.js.map
|
package/dist/browser/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ObservedAction } from "./actions.js";
|
|
2
2
|
import type { PageSnapshot } from "../shared/page-protocol.js";
|
|
3
|
-
export type { SnapshotScope, SnapshotFormat, SemanticState, PageActionCandidate, PageSnapshot } from "../shared/page-protocol.js";
|
|
3
|
+
export type { SnapshotScope, SnapshotFormat, SemanticState, PageActionCandidate, PageSnapshot, } from "../shared/page-protocol.js";
|
|
4
4
|
export interface RefTarget {
|
|
5
5
|
element: Element;
|
|
6
6
|
identity: string;
|
package/dist/browser/types.js
CHANGED
package/dist/definition.d.ts
CHANGED
|
@@ -8,7 +8,5 @@ export declare function isValidAgentContextName(name: string): boolean;
|
|
|
8
8
|
export declare function defineAgentTool<TResult = unknown>(metadata: AgentToolMetadataInput): AgentToolDefinition<Record<string, never>, TResult>;
|
|
9
9
|
export declare function defineAgentTool<TInput = never, TResult = unknown, const TSchema extends AgentToolInputSchema = AgentToolInputSchema>(metadata: AgentToolMetadataInput & {
|
|
10
10
|
inputSchema: TSchema;
|
|
11
|
-
}): AgentToolDefinition<[
|
|
12
|
-
TInput
|
|
13
|
-
] extends [never] ? InferJsonSchema<TSchema> : TInput, TResult>;
|
|
11
|
+
}): AgentToolDefinition<[TInput] extends [never] ? InferJsonSchema<TSchema> : TInput, TResult>;
|
|
14
12
|
export {};
|
package/dist/definition.js
CHANGED
|
@@ -6,7 +6,8 @@ const emptyInputSchema = Object.freeze({
|
|
|
6
6
|
});
|
|
7
7
|
const agentNamePattern = /^[A-Za-z0-9_-]{1,64}$/;
|
|
8
8
|
const reservedContextNames = new Set([
|
|
9
|
-
"nxf_applyAction",
|
|
9
|
+
"nxf_applyAction",
|
|
10
|
+
"nxf_delegatePageTask",
|
|
10
11
|
"nxf_callAppTool",
|
|
11
12
|
"nxf_finish",
|
|
12
13
|
"nxf_getSnapshot",
|
|
@@ -33,4 +34,3 @@ export function defineAgentTool(metadata) {
|
|
|
33
34
|
inputSchema: metadata.inputSchema === undefined ? emptyInputSchema : metadata.inputSchema,
|
|
34
35
|
});
|
|
35
36
|
}
|
|
36
|
-
//# sourceMappingURL=definition.js.map
|
package/dist/index.browser.js
CHANGED
package/dist/index.js
CHANGED
|
@@ -4,4 +4,3 @@ export { AgentOperationSurface } from "./operation-surface.js";
|
|
|
4
4
|
export { createToolRuntime, isToolRuntimeError, } from "./runtime.js";
|
|
5
5
|
export { AgentContext, ToolsProvider, AgentUIAction, useAgentContext, useAgentNavigation, useAgentTool, } from "./react.js";
|
|
6
6
|
export { registerAgentTool } from "./registration.js";
|
|
7
|
-
//# sourceMappingURL=index.js.map
|