@nexface/tools 0.1.13-alpha.4 → 0.1.13-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/actions.js +34 -9
- package/dist/browser/controller.js +53 -73
- package/dist/browser/diagnostics.js +8 -2
- package/dist/browser/dom.js +12 -27
- package/dist/browser/feedback.js +6 -4
- package/dist/browser/locator.js +3 -8
- package/dist/browser/snapshot.js +10 -21
- package/dist/browser/toolset.js +42 -35
- package/dist/browser/types.d.ts +1 -1
- package/dist/definition.d.ts +1 -3
- package/dist/definition.js +2 -1
- package/dist/internal-react.d.ts +1 -1
- package/dist/internal-react.js +3 -3
- package/dist/operation-lock.js +36 -1
- package/dist/operation-runtime.js +41 -12
- package/dist/operation-surface.js +118 -40
- package/dist/operation-view.d.ts +1 -1
- package/dist/operation-view.js +120 -59
- package/dist/react.js +6 -8
- package/dist/registration.js +3 -1
- package/dist/runtime.d.ts +1 -1
- package/dist/runtime.js +66 -44
- package/dist/shared/errors.d.ts +1 -1
- package/dist/shared/schema.js +45 -18
- package/dist/styles.css +879 -109
- package/dist/tool-search.js +37 -11
- package/dist/webmcp.js +38 -11
- package/package.json +1 -1
package/dist/browser/toolset.js
CHANGED
|
@@ -27,11 +27,7 @@ const DEFAULT_SNAPSHOT_LIMITS = {
|
|
|
27
27
|
fullMaxVisitedNodes: 40_000,
|
|
28
28
|
};
|
|
29
29
|
function isIgnoredMutationNode(node) {
|
|
30
|
-
const element = node instanceof Element
|
|
31
|
-
? node
|
|
32
|
-
: node instanceof Text
|
|
33
|
-
? node.parentElement
|
|
34
|
-
: null;
|
|
30
|
+
const element = node instanceof Element ? node : node instanceof Text ? node.parentElement : null;
|
|
35
31
|
return Boolean(element?.closest("[data-nexface-ignore]"));
|
|
36
32
|
}
|
|
37
33
|
function isRelevantMutation(record) {
|
|
@@ -40,8 +36,7 @@ function isRelevantMutation(record) {
|
|
|
40
36
|
if (record.type !== "childList")
|
|
41
37
|
return true;
|
|
42
38
|
const changedNodes = [...record.addedNodes, ...record.removedNodes];
|
|
43
|
-
return
|
|
44
|
-
|| changedNodes.some((node) => !isIgnoredMutationNode(node)));
|
|
39
|
+
return changedNodes.length === 0 || changedNodes.some((node) => !isIgnoredMutationNode(node));
|
|
45
40
|
}
|
|
46
41
|
let activeHistoryPatch;
|
|
47
42
|
function subscribeHistoryChanges(subscriber) {
|
|
@@ -223,12 +218,12 @@ function validateInput(definition, input) {
|
|
|
223
218
|
}
|
|
224
219
|
}
|
|
225
220
|
export function browserToolsAvailable() {
|
|
226
|
-
return (typeof window !== "undefined"
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
221
|
+
return (typeof window !== "undefined" &&
|
|
222
|
+
typeof document !== "undefined" &&
|
|
223
|
+
typeof Worker !== "undefined" &&
|
|
224
|
+
typeof Blob !== "undefined" &&
|
|
225
|
+
typeof URL?.createObjectURL === "function" &&
|
|
226
|
+
typeof MutationObserver !== "undefined");
|
|
232
227
|
}
|
|
233
228
|
export function createBrowserToolset(options = {}) {
|
|
234
229
|
if (!browserToolsAvailable()) {
|
|
@@ -236,19 +231,14 @@ export function createBrowserToolset(options = {}) {
|
|
|
236
231
|
}
|
|
237
232
|
const limits = {
|
|
238
233
|
...DEFAULT_EXECUTION_LIMITS,
|
|
239
|
-
maxDurationMs: options.limits?.maxDurationMs
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
?? DEFAULT_EXECUTION_LIMITS.maxResultChars,
|
|
243
|
-
singleActionTimeoutMs: options.limits?.singleActionTimeoutMs
|
|
244
|
-
?? DEFAULT_EXECUTION_LIMITS.singleActionTimeoutMs,
|
|
234
|
+
maxDurationMs: options.limits?.maxDurationMs ?? DEFAULT_EXECUTION_LIMITS.maxDurationMs,
|
|
235
|
+
maxResultChars: options.limits?.maxResultChars ?? DEFAULT_EXECUTION_LIMITS.maxResultChars,
|
|
236
|
+
singleActionTimeoutMs: options.limits?.singleActionTimeoutMs ?? DEFAULT_EXECUTION_LIMITS.singleActionTimeoutMs,
|
|
245
237
|
};
|
|
246
238
|
const snapshotLimits = {
|
|
247
239
|
...DEFAULT_SNAPSHOT_LIMITS,
|
|
248
|
-
viewportMaxChars: options.snapshotLimits?.viewportMaxChars
|
|
249
|
-
|
|
250
|
-
fullMaxChars: options.snapshotLimits?.fullMaxChars
|
|
251
|
-
?? DEFAULT_SNAPSHOT_LIMITS.fullMaxChars,
|
|
240
|
+
viewportMaxChars: options.snapshotLimits?.viewportMaxChars ?? DEFAULT_SNAPSHOT_LIMITS.viewportMaxChars,
|
|
241
|
+
fullMaxChars: options.snapshotLimits?.fullMaxChars ?? DEFAULT_SNAPSHOT_LIMITS.fullMaxChars,
|
|
252
242
|
};
|
|
253
243
|
const feedback = new ActionFeedback();
|
|
254
244
|
const controller = new PageController(limits, feedback);
|
|
@@ -285,7 +275,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
285
275
|
const scope = normalizedInput.scope;
|
|
286
276
|
const deadline = performance.now() + limits.maxDurationMs;
|
|
287
277
|
signal.throwIfAborted();
|
|
288
|
-
return (await observe(scope, { deadline, signal }, pageDecision ? "actions" : "text"))
|
|
278
|
+
return (await observe(scope, { deadline, signal }, pageDecision ? "actions" : "text"))
|
|
279
|
+
.snapshot;
|
|
289
280
|
},
|
|
290
281
|
};
|
|
291
282
|
const executeObserved = async (signal, execute, cancel, mode = "code", onFeedback, assertActive) => {
|
|
@@ -316,7 +307,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
316
307
|
}
|
|
317
308
|
result = await measurePagePhase("execution", () => execute(activeRegistry, workerBudget));
|
|
318
309
|
signal.throwIfAborted();
|
|
319
|
-
if (result.error &&
|
|
310
|
+
if (result.error &&
|
|
311
|
+
["EXECUTION_TIMEOUT", "WORKER_ERROR", "EXECUTION_OUTCOME_UNKNOWN"].includes(result.error.code)) {
|
|
320
312
|
result.error.code = "EXECUTION_OUTCOME_UNKNOWN";
|
|
321
313
|
}
|
|
322
314
|
if (!result.ok && result.completedActions.length === 0) {
|
|
@@ -325,9 +317,7 @@ export function createBrowserToolset(options = {}) {
|
|
|
325
317
|
try {
|
|
326
318
|
const settled = await measurePagePhase("settle", () => tracker.waitForSettle({ deadline, signal }));
|
|
327
319
|
signal.throwIfAborted();
|
|
328
|
-
if (settled.timedOut
|
|
329
|
-
&& settled.urlChanged
|
|
330
|
-
&& !settled.mutationObservedAfterUrlChange) {
|
|
320
|
+
if (settled.timedOut && settled.urlChanged && !settled.mutationObservedAfterUrlChange) {
|
|
331
321
|
result.snapshotUnavailable = true;
|
|
332
322
|
return expireRefs(result);
|
|
333
323
|
}
|
|
@@ -382,7 +372,9 @@ export function createBrowserToolset(options = {}) {
|
|
|
382
372
|
if (!bundle || bundle.snapshot.snapshotId !== input.snapshotId)
|
|
383
373
|
return undefined;
|
|
384
374
|
const selected = bundle.actions?.get(input.actionId);
|
|
385
|
-
if (!selected ||
|
|
375
|
+
if (!selected ||
|
|
376
|
+
selected.command.method === "scrollBy" ||
|
|
377
|
+
selected.command.method === "waitForTimeout")
|
|
386
378
|
return undefined;
|
|
387
379
|
const target = selected.command.target;
|
|
388
380
|
const element = target?.type === "ref" ? bundle.registry.refs.get(target.ref)?.element : undefined;
|
|
@@ -396,14 +388,21 @@ export function createBrowserToolset(options = {}) {
|
|
|
396
388
|
const selected = bundle.actions.get(input.actionId);
|
|
397
389
|
if (!selected)
|
|
398
390
|
throw new AgentToolError("INVALID_INPUT", "Action is not a candidate of this observation.");
|
|
399
|
-
if (selected.command.method === "fill"
|
|
391
|
+
if (selected.command.method === "fill"
|
|
392
|
+
? typeof input.text !== "string" || !input.text.trim()
|
|
393
|
+
: input.text !== undefined) {
|
|
400
394
|
throw new AgentToolError("INVALID_INPUT", "Only fill actions require a non-empty text value.");
|
|
401
395
|
}
|
|
402
396
|
if (selected.command.method === "selectOption") {
|
|
403
397
|
const target = selected.command.target;
|
|
404
398
|
const element = target.type === "ref" ? bundle.registry.refs.get(target.ref)?.element : undefined;
|
|
405
|
-
const options = element instanceof HTMLSelectElement
|
|
406
|
-
|
|
399
|
+
const options = element instanceof HTMLSelectElement
|
|
400
|
+
? [...element.options].filter((option) => option.value === selected.command.args[0])
|
|
401
|
+
: [];
|
|
402
|
+
if (options.length !== 1 ||
|
|
403
|
+
options[0].disabled ||
|
|
404
|
+
options[0].closest("optgroup[disabled]") ||
|
|
405
|
+
options[0].label.slice(0, 500) !== selected.candidate.option) {
|
|
407
406
|
throw new AgentToolError("INVALID_INPUT", "The observed option changed or is no longer available. Observe again.");
|
|
408
407
|
}
|
|
409
408
|
}
|
|
@@ -413,7 +412,9 @@ export function createBrowserToolset(options = {}) {
|
|
|
413
412
|
let timer;
|
|
414
413
|
try {
|
|
415
414
|
const command = {
|
|
416
|
-
...selected.command,
|
|
415
|
+
...selected.command,
|
|
416
|
+
executionId,
|
|
417
|
+
commandSeq: 1,
|
|
417
418
|
args: selected.command.method === "fill" ? [input.text] : selected.command.args,
|
|
418
419
|
};
|
|
419
420
|
const response = await Promise.race([
|
|
@@ -421,7 +422,8 @@ export function createBrowserToolset(options = {}) {
|
|
|
421
422
|
if (selected.optionElement) {
|
|
422
423
|
const target = selected.command.target;
|
|
423
424
|
const element = target.type === "ref" ? registry.refs.get(target.ref)?.element : undefined;
|
|
424
|
-
if (!selected.optionElement.isConnected ||
|
|
425
|
+
if (!selected.optionElement.isConnected ||
|
|
426
|
+
selected.optionElement.closest("select") !== element) {
|
|
425
427
|
throw new AgentToolError("INVALID_INPUT", "The observed option was replaced. Observe again.");
|
|
426
428
|
}
|
|
427
429
|
}
|
|
@@ -434,7 +436,12 @@ export function createBrowserToolset(options = {}) {
|
|
|
434
436
|
}),
|
|
435
437
|
]);
|
|
436
438
|
const receipt = controller.completeExecution(executionId);
|
|
437
|
-
return {
|
|
439
|
+
return {
|
|
440
|
+
executionId,
|
|
441
|
+
...receipt,
|
|
442
|
+
ok: response.ok,
|
|
443
|
+
...(response.error ? { error: response.error } : {}),
|
|
444
|
+
};
|
|
438
445
|
}
|
|
439
446
|
finally {
|
|
440
447
|
clearTimeout(timer);
|
package/dist/browser/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ObservedAction } from "./actions.js";
|
|
2
2
|
import type { PageSnapshot } from "../shared/page-protocol.js";
|
|
3
|
-
export type { SnapshotScope, SnapshotFormat, SemanticState, PageActionCandidate, PageSnapshot } from "../shared/page-protocol.js";
|
|
3
|
+
export type { SnapshotScope, SnapshotFormat, SemanticState, PageActionCandidate, PageSnapshot, } from "../shared/page-protocol.js";
|
|
4
4
|
export interface RefTarget {
|
|
5
5
|
element: Element;
|
|
6
6
|
identity: string;
|
package/dist/definition.d.ts
CHANGED
|
@@ -8,7 +8,5 @@ export declare function isValidAgentContextName(name: string): boolean;
|
|
|
8
8
|
export declare function defineAgentTool<TResult = unknown>(metadata: AgentToolMetadataInput): AgentToolDefinition<Record<string, never>, TResult>;
|
|
9
9
|
export declare function defineAgentTool<TInput = never, TResult = unknown, const TSchema extends AgentToolInputSchema = AgentToolInputSchema>(metadata: AgentToolMetadataInput & {
|
|
10
10
|
inputSchema: TSchema;
|
|
11
|
-
}): AgentToolDefinition<[
|
|
12
|
-
TInput
|
|
13
|
-
] extends [never] ? InferJsonSchema<TSchema> : TInput, TResult>;
|
|
11
|
+
}): AgentToolDefinition<[TInput] extends [never] ? InferJsonSchema<TSchema> : TInput, TResult>;
|
|
14
12
|
export {};
|
package/dist/definition.js
CHANGED
|
@@ -6,7 +6,8 @@ const emptyInputSchema = Object.freeze({
|
|
|
6
6
|
});
|
|
7
7
|
const agentNamePattern = /^[A-Za-z0-9_-]{1,64}$/;
|
|
8
8
|
const reservedContextNames = new Set([
|
|
9
|
-
"nxf_applyAction",
|
|
9
|
+
"nxf_applyAction",
|
|
10
|
+
"nxf_delegatePageTask",
|
|
10
11
|
"nxf_callAppTool",
|
|
11
12
|
"nxf_finish",
|
|
12
13
|
"nxf_getSnapshot",
|
package/dist/internal-react.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { type ToolRuntime, type InternalToolRuntime } from "./runtime.js";
|
|
|
3
3
|
export interface InternalToolRuntimeHostProps {
|
|
4
4
|
children: ReactNode;
|
|
5
5
|
}
|
|
6
|
-
export declare function InternalToolRuntimeHost({ children
|
|
6
|
+
export declare function InternalToolRuntimeHost({ children }: InternalToolRuntimeHostProps): ReactElement;
|
|
7
7
|
export declare function InternalToolRuntimeScope({ children, runtime, }: {
|
|
8
8
|
children: ReactNode;
|
|
9
9
|
runtime: ToolRuntime;
|
package/dist/internal-react.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import * as React from "react";
|
|
2
|
-
import { createContext, useContext, useMemo
|
|
2
|
+
import { createContext, useContext, useMemo } from "react";
|
|
3
3
|
import { createToolRuntime, getInternalToolRuntime, } from "./runtime.js";
|
|
4
4
|
const InternalToolRuntimeContext = createContext(null);
|
|
5
|
-
export function InternalToolRuntimeHost({ children
|
|
5
|
+
export function InternalToolRuntimeHost({ children }) {
|
|
6
6
|
const runtime = useMemo(() => createToolRuntime(), []);
|
|
7
|
-
return
|
|
7
|
+
return React.createElement(InternalToolRuntimeScope, { runtime: runtime }, children);
|
|
8
8
|
}
|
|
9
9
|
export function InternalToolRuntimeScope({ children, runtime, }) {
|
|
10
10
|
const internalRuntime = getInternalToolRuntime(runtime);
|
package/dist/operation-lock.js
CHANGED
|
@@ -30,7 +30,42 @@ export function lockOperationSurface(surface, redirectFocus) {
|
|
|
30
30
|
event.stopImmediatePropagation();
|
|
31
31
|
};
|
|
32
32
|
// Wheel and touch defaults are blocked here: React delegates them passively.
|
|
33
|
-
const events = [
|
|
33
|
+
const events = [
|
|
34
|
+
"focus",
|
|
35
|
+
"focusin",
|
|
36
|
+
"mouseenter",
|
|
37
|
+
"mouseleave",
|
|
38
|
+
"pointerenter",
|
|
39
|
+
"pointerleave",
|
|
40
|
+
"pointerdown",
|
|
41
|
+
"pointerup",
|
|
42
|
+
"pointermove",
|
|
43
|
+
"pointerover",
|
|
44
|
+
"pointerout",
|
|
45
|
+
"mousedown",
|
|
46
|
+
"mouseup",
|
|
47
|
+
"mousemove",
|
|
48
|
+
"mouseover",
|
|
49
|
+
"mouseout",
|
|
50
|
+
"click",
|
|
51
|
+
"dblclick",
|
|
52
|
+
"auxclick",
|
|
53
|
+
"contextmenu",
|
|
54
|
+
"keydown",
|
|
55
|
+
"keyup",
|
|
56
|
+
"beforeinput",
|
|
57
|
+
"paste",
|
|
58
|
+
"cut",
|
|
59
|
+
"drop",
|
|
60
|
+
"dragstart",
|
|
61
|
+
"touchstart",
|
|
62
|
+
"touchmove",
|
|
63
|
+
"touchend",
|
|
64
|
+
"touchcancel",
|
|
65
|
+
"wheel",
|
|
66
|
+
"submit",
|
|
67
|
+
"reset",
|
|
68
|
+
];
|
|
34
69
|
lockedSurfaces.add(surface);
|
|
35
70
|
surface.setAttribute("aria-busy", "true");
|
|
36
71
|
for (const name of events)
|
|
@@ -4,17 +4,24 @@ export class OperationTracker {
|
|
|
4
4
|
#unscoped = 0;
|
|
5
5
|
#executions = new Map();
|
|
6
6
|
#listeners = new Set();
|
|
7
|
-
#snapshot = Object.freeze({
|
|
7
|
+
#snapshot = Object.freeze({
|
|
8
|
+
operation: null,
|
|
9
|
+
executions: Object.freeze([]),
|
|
10
|
+
});
|
|
8
11
|
getSnapshot = () => this.#snapshot;
|
|
9
12
|
subscribe = (listener) => {
|
|
10
13
|
this.#listeners.add(listener);
|
|
11
|
-
return () => {
|
|
14
|
+
return () => {
|
|
15
|
+
this.#listeners.delete(listener);
|
|
16
|
+
};
|
|
12
17
|
};
|
|
13
18
|
begin(options = {}) {
|
|
14
19
|
options.signal?.throwIfAborted();
|
|
15
20
|
const previous = this.#operation;
|
|
16
|
-
const replacing = previous &&
|
|
17
|
-
|
|
21
|
+
const replacing = previous &&
|
|
22
|
+
previous.handle === options.replaceWaiting &&
|
|
23
|
+
previous.snapshot.status === "waiting" &&
|
|
24
|
+
previous.pending === 0;
|
|
18
25
|
if (this.#unscoped || (previous && !replacing)) {
|
|
19
26
|
throw new AgentToolError("OPERATION_BUSY", "The Runtime already has an operation or outstanding tool calls.");
|
|
20
27
|
}
|
|
@@ -23,11 +30,22 @@ export class OperationTracker {
|
|
|
23
30
|
previous.resolveEnd();
|
|
24
31
|
}
|
|
25
32
|
let resolveEnd;
|
|
26
|
-
const endPromise = new Promise((resolve) => {
|
|
33
|
+
const endPromise = new Promise((resolve) => {
|
|
34
|
+
resolveEnd = resolve;
|
|
35
|
+
});
|
|
27
36
|
const id = globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`;
|
|
28
37
|
const operation = {
|
|
29
|
-
snapshot: Object.freeze({
|
|
30
|
-
|
|
38
|
+
snapshot: Object.freeze({
|
|
39
|
+
id,
|
|
40
|
+
source: options.source ?? "host",
|
|
41
|
+
status: "running",
|
|
42
|
+
hasExecuted: false,
|
|
43
|
+
canCancel: Boolean(options.onCancel),
|
|
44
|
+
...(options.taskProgress ? { taskProgress: Object.freeze(options.taskProgress) } : {}),
|
|
45
|
+
}),
|
|
46
|
+
handle: undefined,
|
|
47
|
+
pending: 0,
|
|
48
|
+
resolveEnd,
|
|
31
49
|
...(options.onCancel ? { onCancel: options.onCancel } : {}),
|
|
32
50
|
removeAbort: () => options.signal?.removeEventListener("abort", onAbort),
|
|
33
51
|
};
|
|
@@ -42,7 +60,11 @@ export class OperationTracker {
|
|
|
42
60
|
if (this.#operation !== operation || operation.snapshot.status === "ending")
|
|
43
61
|
return endPromise;
|
|
44
62
|
operation.removeAbort();
|
|
45
|
-
operation.snapshot = Object.freeze({
|
|
63
|
+
operation.snapshot = Object.freeze({
|
|
64
|
+
...operation.snapshot,
|
|
65
|
+
status: "ending",
|
|
66
|
+
outcome: endOptions?.outcome ?? "completed",
|
|
67
|
+
});
|
|
46
68
|
this.#publish();
|
|
47
69
|
if (this.#finish(operation))
|
|
48
70
|
this.#publish();
|
|
@@ -77,8 +99,11 @@ export class OperationTracker {
|
|
|
77
99
|
}
|
|
78
100
|
requestCancel(id) {
|
|
79
101
|
const operation = this.#operation;
|
|
80
|
-
if (!operation ||
|
|
81
|
-
|
|
102
|
+
if (!operation ||
|
|
103
|
+
operation.snapshot.id !== id ||
|
|
104
|
+
!operation.onCancel ||
|
|
105
|
+
operation.snapshot.status === "ending" ||
|
|
106
|
+
operation.snapshot.status === "cancelling")
|
|
82
107
|
return;
|
|
83
108
|
const previous = operation.snapshot.status;
|
|
84
109
|
this.#update(operation, { status: "cancelling" });
|
|
@@ -123,7 +148,9 @@ export class OperationTracker {
|
|
|
123
148
|
if (event.status === "running") {
|
|
124
149
|
this.#executions.set(event.id, event);
|
|
125
150
|
const operation = this.#operation;
|
|
126
|
-
if (operation &&
|
|
151
|
+
if (operation &&
|
|
152
|
+
event.operationId === operation.snapshot.id &&
|
|
153
|
+
!operation.snapshot.hasExecuted) {
|
|
127
154
|
operation.snapshot = Object.freeze({ ...operation.snapshot, hasExecuted: true });
|
|
128
155
|
}
|
|
129
156
|
}
|
|
@@ -132,7 +159,9 @@ export class OperationTracker {
|
|
|
132
159
|
this.#publish();
|
|
133
160
|
}
|
|
134
161
|
#finish(operation) {
|
|
135
|
-
if (operation.pending ||
|
|
162
|
+
if (operation.pending ||
|
|
163
|
+
operation.snapshot.status !== "ending" ||
|
|
164
|
+
this.#operation !== operation)
|
|
136
165
|
return false;
|
|
137
166
|
this.#operation = undefined;
|
|
138
167
|
operation.resolveEnd();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as React from "react";
|
|
2
2
|
import { flushSync } from "react-dom";
|
|
3
|
-
import { useCallback, useLayoutEffect, useRef, useState } from "react";
|
|
3
|
+
import { useCallback, useLayoutEffect, useRef, useState, } from "react";
|
|
4
4
|
import { useSyncExternalStore } from "use-sync-external-store/shim";
|
|
5
5
|
import { PageCommandError } from "./browser/locator.js";
|
|
6
6
|
import { getInternalToolRuntime } from "./runtime.js";
|
|
@@ -9,7 +9,8 @@ import { isAgentActivation } from "./browser/focus.js";
|
|
|
9
9
|
import { OperationOverlayView } from "./operation-view.js";
|
|
10
10
|
const BROWSER_TOOL_NAMES = new Set(["nxf_getSnapshot", "nxf_executeAction", "nxf_applyAction"]);
|
|
11
11
|
const belongsToOperation = (event, operation) => !operation || event.operationId === operation.id;
|
|
12
|
-
const canPresentInOperation = (event, operation) => belongsToOperation(event, operation) &&
|
|
12
|
+
const canPresentInOperation = (event, operation) => belongsToOperation(event, operation) &&
|
|
13
|
+
(!operation || operation.status === "running" || operation.status === "ending");
|
|
13
14
|
const surfaces = new WeakSet();
|
|
14
15
|
export function AgentOperationSurface(props) {
|
|
15
16
|
const element = useRef(null);
|
|
@@ -21,28 +22,34 @@ export function AgentOperationSurface(props) {
|
|
|
21
22
|
if (surfaces.has(props.toolRuntime))
|
|
22
23
|
throw new Error("Only one AgentOperationSurface is supported per Runtime.");
|
|
23
24
|
surfaces.add(props.toolRuntime);
|
|
24
|
-
return () => {
|
|
25
|
+
return () => {
|
|
26
|
+
surfaces.delete(props.toolRuntime);
|
|
27
|
+
};
|
|
25
28
|
}, [props.toolRuntime]);
|
|
26
29
|
const blockUserInput = useCallback((event) => {
|
|
27
30
|
if ((event.type === "submit" || event.type === "reset") && isAgentActivation())
|
|
28
31
|
return;
|
|
29
|
-
if (!element.current ||
|
|
32
|
+
if (!element.current ||
|
|
33
|
+
!isOperationSurfaceLocked(element.current) ||
|
|
34
|
+
!event.nativeEvent.isTrusted)
|
|
30
35
|
return;
|
|
31
36
|
event.preventDefault();
|
|
32
37
|
event.stopPropagation();
|
|
33
38
|
}, []);
|
|
34
39
|
const blockPassiveUserInput = useCallback((event) => {
|
|
35
|
-
if (!element.current ||
|
|
40
|
+
if (!element.current ||
|
|
41
|
+
!isOperationSurfaceLocked(element.current) ||
|
|
42
|
+
!event.nativeEvent.isTrusted)
|
|
36
43
|
return;
|
|
37
44
|
// Stop React's capture dispatch, but let the native event reach our non-passive
|
|
38
45
|
// surface listener. React's passive root listener cannot prevent scrolling.
|
|
39
46
|
event.isPropagationStopped = () => true;
|
|
40
47
|
}, []);
|
|
41
|
-
return React.createElement(React.Fragment, null,
|
|
48
|
+
return (React.createElement(React.Fragment, null,
|
|
42
49
|
React.createElement("div", { ref: element, className: "nexface-operation-surface", onClickCapture: blockUserInput, onDoubleClickCapture: blockUserInput, onAuxClickCapture: blockUserInput, onPointerDownCapture: blockUserInput, onPointerUpCapture: blockUserInput, onPointerMoveCapture: blockUserInput, onPointerOverCapture: blockUserInput, onPointerOutCapture: blockUserInput, onMouseDownCapture: blockUserInput, onMouseUpCapture: blockUserInput, onContextMenuCapture: blockUserInput, onMouseMoveCapture: blockUserInput, onMouseOverCapture: blockUserInput, onMouseOutCapture: blockUserInput, onWheelCapture: blockPassiveUserInput, onTouchStartCapture: blockPassiveUserInput, onTouchMoveCapture: blockPassiveUserInput, onTouchEndCapture: blockUserInput, onTouchCancelCapture: blockUserInput, onKeyDownCapture: blockUserInput, onKeyUpCapture: blockUserInput, onKeyPressCapture: blockUserInput, onBeforeInputCapture: blockUserInput, onPasteCapture: blockUserInput, onCutCapture: blockUserInput, onDropCapture: blockUserInput, onDragStartCapture: blockUserInput, onSubmitCapture: blockUserInput, onResetCapture: blockUserInput }, props.children),
|
|
43
|
-
React.createElement(OperationSurfaceContent, { ...props, element: element }));
|
|
50
|
+
React.createElement(OperationSurfaceContent, { ...props, element: element })));
|
|
44
51
|
}
|
|
45
|
-
function OperationSurfaceContent({ toolRuntime, element }) {
|
|
52
|
+
function OperationSurfaceContent({ toolRuntime, element, }) {
|
|
46
53
|
const runtime = getInternalToolRuntime(toolRuntime);
|
|
47
54
|
const activity = useSyncExternalStore(runtime.subscribeActivity, runtime.getActivitySnapshot, runtime.getActivitySnapshot);
|
|
48
55
|
const cycle = activity.operation;
|
|
@@ -58,15 +65,24 @@ function OperationSurfaceContent({ toolRuntime, element }) {
|
|
|
58
65
|
useLayoutEffect(() => runtime.setPageActionFeedback((event, feedback, signal) => {
|
|
59
66
|
const operation = runtime.getActivitySnapshot().operation;
|
|
60
67
|
if (signal?.aborted || !canPresentInOperation(event, operation))
|
|
61
|
-
throw new PageCommandError({
|
|
68
|
+
throw new PageCommandError({
|
|
69
|
+
code: "EXECUTION_CANCELLED",
|
|
70
|
+
message: "Page action presentation was interrupted.",
|
|
71
|
+
});
|
|
62
72
|
const target = feedback.target;
|
|
63
|
-
if (!feedback.effect ||
|
|
73
|
+
if (!feedback.effect ||
|
|
74
|
+
(feedback.effect === "action" &&
|
|
75
|
+
(!target?.isConnected || !element.current?.contains(target)))) {
|
|
64
76
|
setVisual(undefined);
|
|
65
77
|
return;
|
|
66
78
|
}
|
|
67
79
|
const next = {
|
|
68
|
-
id: crypto.randomUUID(),
|
|
69
|
-
|
|
80
|
+
id: crypto.randomUUID(),
|
|
81
|
+
effect: feedback.effect,
|
|
82
|
+
label: label(event),
|
|
83
|
+
centered: feedback.effect === "read",
|
|
84
|
+
persistentCursor: true,
|
|
85
|
+
requiresTarget: feedback.effect === "action",
|
|
70
86
|
...(target ? { target } : {}),
|
|
71
87
|
};
|
|
72
88
|
if (feedback.effect !== "action") {
|
|
@@ -98,32 +114,61 @@ function OperationSurfaceContent({ toolRuntime, element }) {
|
|
|
98
114
|
const page = BROWSER_TOOL_NAMES.has(event.name);
|
|
99
115
|
if (canDisplay && event.status === "running") {
|
|
100
116
|
if (page)
|
|
101
|
-
setVisual(event.name === "nxf_getSnapshot"
|
|
102
|
-
|
|
103
|
-
|
|
117
|
+
setVisual(event.name === "nxf_getSnapshot"
|
|
118
|
+
? {
|
|
119
|
+
event,
|
|
120
|
+
feedback: {
|
|
121
|
+
id: event.id,
|
|
122
|
+
effect: "read",
|
|
123
|
+
label: label(event),
|
|
124
|
+
centered: true,
|
|
125
|
+
},
|
|
126
|
+
}
|
|
127
|
+
: undefined);
|
|
104
128
|
else {
|
|
105
|
-
const target = event.tool.definition.effect === "navigate"
|
|
129
|
+
const target = event.tool.definition.effect === "navigate"
|
|
130
|
+
? element.current
|
|
131
|
+
: runtime.resolveTarget(event.tool, event.input);
|
|
106
132
|
if (target?.isConnected && element.current?.contains(target)) {
|
|
107
133
|
const rect = target.getBoundingClientRect();
|
|
108
|
-
setVisual({
|
|
109
|
-
|
|
110
|
-
|
|
134
|
+
setVisual({
|
|
135
|
+
event,
|
|
136
|
+
feedback: {
|
|
137
|
+
id: event.id,
|
|
138
|
+
effect: event.tool.definition.effect ?? "action",
|
|
139
|
+
label: label(event),
|
|
140
|
+
resolveTarget: () => event.tool.definition.effect === "navigate"
|
|
141
|
+
? (element.current ?? undefined)
|
|
142
|
+
: runtime.resolveTarget(event.tool, event.input),
|
|
143
|
+
position: { left: rect.left + rect.width / 2, top: rect.top + rect.height / 2 },
|
|
144
|
+
},
|
|
145
|
+
});
|
|
111
146
|
}
|
|
112
147
|
}
|
|
113
148
|
}
|
|
114
149
|
else if (canDisplay)
|
|
115
150
|
setVisual((current) => current?.event.id === event.id && current.event.operationId === event.operationId
|
|
116
|
-
? page
|
|
151
|
+
? page
|
|
152
|
+
? undefined
|
|
153
|
+
: { ...current, event }
|
|
154
|
+
: current);
|
|
117
155
|
if (page && !operation && event.status === "error")
|
|
118
156
|
setOutcome({ id: scope, status: event.cancelled ? "cancelled" : "failed" });
|
|
119
157
|
setHistory((current) => {
|
|
120
|
-
if (!belongsToOperation(event, operation) ||
|
|
158
|
+
if (!belongsToOperation(event, operation) ||
|
|
159
|
+
(event.status !== "running" && current.scope !== scope))
|
|
121
160
|
return current;
|
|
122
161
|
const entries = current.scope === scope ? current.entries : [];
|
|
123
|
-
const entry = {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
162
|
+
const entry = {
|
|
163
|
+
toolCallId: event.id,
|
|
164
|
+
progressLabel: `${event.progressLabel || "正在处理你的请求"} · ${event.cancelled ? "已取消" : event.status === "running" ? "执行中" : event.status === "success" ? "已完成" : "失败"}`,
|
|
165
|
+
};
|
|
166
|
+
return {
|
|
167
|
+
scope,
|
|
168
|
+
entries: entries.some((item) => item.toolCallId === event.id)
|
|
169
|
+
? entries.map((item) => (item.toolCallId === event.id ? entry : item))
|
|
170
|
+
: [...entries, entry],
|
|
171
|
+
};
|
|
127
172
|
});
|
|
128
173
|
};
|
|
129
174
|
const unsubscribe = toolRuntime.on("toolexecution", onExecution);
|
|
@@ -136,24 +181,57 @@ function OperationSurfaceContent({ toolRuntime, element }) {
|
|
|
136
181
|
const waiting = cycle?.status === "waiting" && !executing;
|
|
137
182
|
const running = executing || (activated && cycle?.status !== "waiting");
|
|
138
183
|
const stopping = cycle?.status === "cancelling";
|
|
139
|
-
useLayoutEffect(() => {
|
|
140
|
-
|
|
184
|
+
useLayoutEffect(() => {
|
|
185
|
+
if (!running || waiting)
|
|
186
|
+
setVisual(undefined);
|
|
187
|
+
}, [running, waiting]);
|
|
141
188
|
const serverExecuting = cycle?.taskProgress?.phase === "tool";
|
|
142
189
|
const visualExecuting = Boolean(visual && activity.executions.some((event) => event.id === visual.event.id));
|
|
143
|
-
const currentVisual = !stopping &&
|
|
144
|
-
|
|
145
|
-
|
|
190
|
+
const currentVisual = !stopping &&
|
|
191
|
+
!waiting &&
|
|
192
|
+
visual &&
|
|
193
|
+
(!serverExecuting || (BROWSER_TOOL_NAMES.has(visual.event.name) && visualExecuting)) &&
|
|
194
|
+
canPresentInOperation(visual.event, cycle) &&
|
|
195
|
+
(cycle || visualExecuting)
|
|
196
|
+
? visual
|
|
197
|
+
: undefined;
|
|
146
198
|
const browserExecution = activity.executions.find((event) => BROWSER_TOOL_NAMES.has(event.name));
|
|
147
|
-
const progressLabel = currentVisual?.feedback.label ??
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
199
|
+
const progressLabel = currentVisual?.feedback.label ??
|
|
200
|
+
(waiting
|
|
201
|
+
? "等待你完成操作"
|
|
202
|
+
: stopping
|
|
203
|
+
? "正在终止"
|
|
204
|
+
: browserExecution
|
|
205
|
+
? browserExecution.progressLabel || "正在浏览网页"
|
|
206
|
+
: cycle?.taskProgress?.phase === "tool"
|
|
207
|
+
? cycle.taskProgress.progressLabel || "正在执行工具"
|
|
208
|
+
: cycle?.taskProgress
|
|
209
|
+
? "正在处理"
|
|
210
|
+
: "正在处理你的请求");
|
|
211
|
+
const operation = {
|
|
212
|
+
progressLabel,
|
|
213
|
+
status: currentVisual
|
|
214
|
+
? currentVisual.event.status === "running"
|
|
215
|
+
? "running"
|
|
216
|
+
: currentVisual.event.status === "success"
|
|
217
|
+
? "completed"
|
|
218
|
+
: "failed"
|
|
219
|
+
: running || waiting
|
|
220
|
+
? "running"
|
|
221
|
+
: outcome?.id === history.scope
|
|
222
|
+
? outcome.status
|
|
223
|
+
: "completed",
|
|
224
|
+
};
|
|
225
|
+
const feedback = currentVisual?.feedback ??
|
|
226
|
+
(serverExecuting && running && !waiting && !stopping
|
|
227
|
+
? {
|
|
228
|
+
id: cycle.taskProgress.toolCallId,
|
|
229
|
+
effect: "read",
|
|
230
|
+
label: progressLabel,
|
|
231
|
+
centered: true,
|
|
232
|
+
}
|
|
233
|
+
: undefined);
|
|
156
234
|
const resolveSurface = useCallback(() => element.current ?? undefined, [element]);
|
|
157
235
|
const entries = !cycle || history.scope === cycle.id ? history.entries : [];
|
|
158
|
-
return activated || executing || entries.length ? React.createElement(OperationOverlayView, { key: cycle?.id ?? history.scope, operation: operation, progressHistory: entries, running: running, humanHasControl: waiting, cancelling: stopping, ...(feedback ? { feedback } : {}), ...(cycle?.canCancel ? { onCancel: () => runtime.requestOperationCancel(cycle.id) } : {}), resolveSurface: resolveSurface }) : null;
|
|
236
|
+
return activated || executing || entries.length ? (React.createElement(OperationOverlayView, { key: cycle?.id ?? history.scope, operation: operation, progressHistory: entries, running: running, humanHasControl: waiting, cancelling: stopping, ...(feedback ? { feedback } : {}), ...(cycle?.canCancel ? { onCancel: () => runtime.requestOperationCancel(cycle.id) } : {}), resolveSurface: resolveSurface })) : null;
|
|
159
237
|
}
|
package/dist/operation-view.d.ts
CHANGED
|
@@ -10,4 +10,4 @@ export interface OperationOverlayViewProps {
|
|
|
10
10
|
onCancel?(): void;
|
|
11
11
|
resolveSurface?(): HTMLElement | undefined;
|
|
12
12
|
}
|
|
13
|
-
export declare function OperationOverlayView({ operation, feedback, progressHistory, running, humanHasControl, cancelling, onCancel: cancel, resolveSurface }: OperationOverlayViewProps): React.ReactPortal | null;
|
|
13
|
+
export declare function OperationOverlayView({ operation, feedback, progressHistory, running, humanHasControl, cancelling, onCancel: cancel, resolveSurface, }: OperationOverlayViewProps): React.ReactPortal | null;
|