@ap3x/browser-agent 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +34 -0
- package/dist/agent/agent.d.ts +160 -0
- package/dist/agent/agent.d.ts.map +1 -0
- package/dist/agent/backend.d.ts +38 -0
- package/dist/agent/backend.d.ts.map +1 -0
- package/dist/agent/captcha-watchdog.d.ts +31 -0
- package/dist/agent/captcha-watchdog.d.ts.map +1 -0
- package/dist/agent/context-builder.d.ts +24 -0
- package/dist/agent/context-builder.d.ts.map +1 -0
- package/dist/agent/event-stream.d.ts +104 -0
- package/dist/agent/event-stream.d.ts.map +1 -0
- package/dist/agent/executor.d.ts +37 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/gif.d.ts +23 -0
- package/dist/agent/gif.d.ts.map +1 -0
- package/dist/agent/history.d.ts +89 -0
- package/dist/agent/history.d.ts.map +1 -0
- package/dist/agent/judge.d.ts +36 -0
- package/dist/agent/judge.d.ts.map +1 -0
- package/dist/agent/message-manager.d.ts +94 -0
- package/dist/agent/message-manager.d.ts.map +1 -0
- package/dist/agent/output-schema.d.ts +26 -0
- package/dist/agent/output-schema.d.ts.map +1 -0
- package/dist/agent/png-decode.d.ts +9 -0
- package/dist/agent/png-decode.d.ts.map +1 -0
- package/dist/agent/rematch.d.ts +26 -0
- package/dist/agent/rematch.d.ts.map +1 -0
- package/dist/agent/replay.d.ts +155 -0
- package/dist/agent/replay.d.ts.map +1 -0
- package/dist/agent/step-loop.d.ts +124 -0
- package/dist/agent/step-loop.d.ts.map +1 -0
- package/dist/agent/variable-detector.d.ts +19 -0
- package/dist/agent/variable-detector.d.ts.map +1 -0
- package/dist/agent/video.d.ts +12 -0
- package/dist/agent/video.d.ts.map +1 -0
- package/dist/chunk-32HNRAQS.js +82 -0
- package/dist/chunk-UZALZUEE.js +2907 -0
- package/dist/filesystem/external-read.d.ts +18 -0
- package/dist/filesystem/external-read.d.ts.map +1 -0
- package/dist/filesystem/file-system.d.ts +61 -0
- package/dist/filesystem/file-system.d.ts.map +1 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1070 -0
- package/dist/mcp/bin.d.ts +3 -0
- package/dist/mcp/bin.d.ts.map +1 -0
- package/dist/mcp/bin.js +19 -0
- package/dist/mcp/client.d.ts +45 -0
- package/dist/mcp/client.d.ts.map +1 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/log.d.ts +6 -0
- package/dist/mcp/log.d.ts.map +1 -0
- package/dist/mcp/server.d.ts +36 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/observability/logger.d.ts +25 -0
- package/dist/observability/logger.d.ts.map +1 -0
- package/dist/testing.d.ts +37 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +18 -0
- package/dist/tools/actions.d.ts +9 -0
- package/dist/tools/actions.d.ts.map +1 -0
- package/dist/tools/registry.d.ts +125 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/sensitive.d.ts +59 -0
- package/dist/tools/sensitive.d.ts.map +1 -0
- package/dist/tools/types.d.ts +117 -0
- package/dist/tools/types.d.ts.map +1 -0
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/usage/ledger.d.ts +27 -0
- package/dist/usage/ledger.d.ts.map +1 -0
- package/package.json +50 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 AP3X
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# @ap3x/browser-agent
|
|
2
|
+
|
|
3
|
+
The agent that drives a browser: a step loop, an action registry, run history, and replay — plus an MCP server exposing it to any MCP client.
|
|
4
|
+
|
|
5
|
+
Part of [AP3X](https://github.com/AP3X-Dev/AP3X) — a TypeScript multi-agent framework.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @ap3x/browser-agent
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## What's inside
|
|
14
|
+
|
|
15
|
+
- **Step loop** — perceive, decide, act, with a structured output schema and a multi-action executor.
|
|
16
|
+
- **Action registry** — `defineAction` with an explicit context object; actions are filtered per page.
|
|
17
|
+
- **Sensitive data** — a placeholder pipeline that keeps secrets out of model context.
|
|
18
|
+
- **FileSystem workspace** — a contained workspace with PDF and DOCX reads.
|
|
19
|
+
- **Run history + replay** — record a run, then replay it deterministically with element re-matching.
|
|
20
|
+
- **Captcha watchdog** — page-state detection that emits a typed signal; it never attempts to solve.
|
|
21
|
+
|
|
22
|
+
## MCP server
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
npx ap3x-browser-mcp
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Documentation
|
|
29
|
+
|
|
30
|
+
See the [AP3X repository](https://github.com/AP3X-Dev/AP3X) for architecture notes, the full package map, and examples.
|
|
31
|
+
|
|
32
|
+
## License
|
|
33
|
+
|
|
34
|
+
MIT
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import type { Api, Model, ProviderStreamOptions } from "@ap3x/ai";
|
|
2
|
+
import { BrowserDriver, type BrowserSession, type HarLog } from "@ap3x/browser";
|
|
3
|
+
import { type Logger } from "../observability/logger";
|
|
4
|
+
import { type ExtractionLlm, type Registry } from "../tools/registry";
|
|
5
|
+
import type { SensitiveData } from "../tools/sensitive";
|
|
6
|
+
import type { VisionMode } from "./context-builder";
|
|
7
|
+
import { type AgentStreamEvent, type AgentStreamEventOf, type AgentStreamEventType } from "./event-stream";
|
|
8
|
+
import { type GifOptions } from "./gif";
|
|
9
|
+
import { AgentHistoryList } from "./history";
|
|
10
|
+
import { type JudgementResult } from "./judge";
|
|
11
|
+
import { type RunOptions, type StepLoopHooks, type StepLoopLimits } from "./step-loop";
|
|
12
|
+
import { type VideoOptions } from "./video";
|
|
13
|
+
/** Grouped constructor options (E1). Only `llm` and `browser` are required. */
|
|
14
|
+
export interface AgentOptions {
|
|
15
|
+
llm: {
|
|
16
|
+
model: Model<Api>;
|
|
17
|
+
/** Optional extraction LLM for the `extract_content` action. */
|
|
18
|
+
extraction?: ExtractionLlm;
|
|
19
|
+
/**
|
|
20
|
+
* Per-call overrides (apiKey/headers/etc.) merged into every LLM call the
|
|
21
|
+
* step loop makes. The seam an embedding host (one that authenticates
|
|
22
|
+
* with per-request credentials rather than a static env key) uses to
|
|
23
|
+
* inject auth without a parallel config shape — this is exactly
|
|
24
|
+
* `@ap3x/ai`'s own `StreamOptions` surface, passed straight through to
|
|
25
|
+
* `generateObject`.
|
|
26
|
+
*/
|
|
27
|
+
streamOptions?: ProviderStreamOptions;
|
|
28
|
+
};
|
|
29
|
+
browser: {
|
|
30
|
+
/** An existing session to drive. */
|
|
31
|
+
session: BrowserSession;
|
|
32
|
+
/** Override the driver built from `session`. */
|
|
33
|
+
driver?: BrowserDriver;
|
|
34
|
+
};
|
|
35
|
+
/** Custom action registry. NEVER mutated — the Agent drives a clone. */
|
|
36
|
+
tools?: Registry;
|
|
37
|
+
/** Screenshot policy (E7). Default `everyStep`. */
|
|
38
|
+
vision?: VisionMode;
|
|
39
|
+
output?: {
|
|
40
|
+
/** Include the `thinking` field (default true). */
|
|
41
|
+
thinking?: boolean;
|
|
42
|
+
/** Minimal `memory`+`action` schema (overrides `thinking`). */
|
|
43
|
+
flash?: boolean;
|
|
44
|
+
/** Build an animated GIF run-summary from history screenshots after `run()` (IMPROVEMENTS GIF). */
|
|
45
|
+
generateGif?: boolean | GifOptions;
|
|
46
|
+
/** Capture network traffic into a HAR 1.2 artifact for the run (M11). Off by default. */
|
|
47
|
+
recordHar?: boolean;
|
|
48
|
+
/** Capture a CDP screencast and assemble it into a video-as-GIF artifact (M11). Off by default. */
|
|
49
|
+
recordVideo?: boolean | VideoOptions;
|
|
50
|
+
};
|
|
51
|
+
limits?: Partial<StepLoopLimits>;
|
|
52
|
+
security?: {
|
|
53
|
+
sensitiveData?: SensitiveData;
|
|
54
|
+
availableFilePaths?: string[];
|
|
55
|
+
};
|
|
56
|
+
hooks?: StepLoopHooks;
|
|
57
|
+
/** Base dir for the FileSystem workspace (default: an OS-temp dir). */
|
|
58
|
+
fileSystemDir?: string;
|
|
59
|
+
/** Replace the default system prompt entirely. */
|
|
60
|
+
systemPrompt?: string;
|
|
61
|
+
/** Injectable structured logger (O1). Default: silent (no output). */
|
|
62
|
+
logger?: Logger;
|
|
63
|
+
/** Highlight each action's element before it runs (IMPROVEMENTS demo-mode overlay). */
|
|
64
|
+
demoMode?: boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Opt-in post-run judge (design/04b §4.5): after `run()` completes, an
|
|
67
|
+
* auxiliary LLM scores the finished trajectory and the verdict is exposed on
|
|
68
|
+
* {@link Agent.judgement}. Informational only — it NEVER overrides the agent's
|
|
69
|
+
* own reported success, and a judge failure is logged-and-swallowed (it never
|
|
70
|
+
* fails the run). Off by default (it's an extra LLM call). Pass `true` to judge
|
|
71
|
+
* with the agent's own model, or an object to override the judge model / supply
|
|
72
|
+
* ground-truth / tune screenshot attachment. The judge's token spend is folded
|
|
73
|
+
* into the run's usage totals.
|
|
74
|
+
*/
|
|
75
|
+
judge?: boolean | {
|
|
76
|
+
/** Judge with a different model than the agent runs with (default: the agent's own model). */
|
|
77
|
+
model?: Model<Api>;
|
|
78
|
+
/** Ground-truth criteria / expected answer; takes precedence over every other signal the judge weighs. */
|
|
79
|
+
groundTruth?: string;
|
|
80
|
+
/** Attach up to this many of the run's most recent screenshots (default 10). */
|
|
81
|
+
maxImages?: number;
|
|
82
|
+
/** Attach screenshots to the judge at all (default true). */
|
|
83
|
+
vision?: boolean;
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export declare class Agent {
|
|
87
|
+
private readonly task;
|
|
88
|
+
private readonly options;
|
|
89
|
+
private readonly model;
|
|
90
|
+
private readonly session;
|
|
91
|
+
private readonly driver;
|
|
92
|
+
private readonly registry;
|
|
93
|
+
private readonly vision;
|
|
94
|
+
private readonly variant;
|
|
95
|
+
private readonly limits;
|
|
96
|
+
private readonly ledger;
|
|
97
|
+
private readonly historyList;
|
|
98
|
+
private readonly messageManager;
|
|
99
|
+
private readonly fileSystemDir;
|
|
100
|
+
private readonly stream;
|
|
101
|
+
private readonly logger;
|
|
102
|
+
private readonly taskId;
|
|
103
|
+
private fileSystem?;
|
|
104
|
+
private stepLoop?;
|
|
105
|
+
private pendingPaused;
|
|
106
|
+
private pendingStopped;
|
|
107
|
+
/** Set once, after `run()` completes, when `output.generateGif` is on. */
|
|
108
|
+
private lastGif?;
|
|
109
|
+
/** Set once, after `run()` completes, when `output.recordHar` is on. */
|
|
110
|
+
private lastHar?;
|
|
111
|
+
/** Set once, after `run()` completes, when `output.recordVideo` is on. */
|
|
112
|
+
private lastVideo?;
|
|
113
|
+
/** Set once, after `run()` completes, when `judge` is on and the judge succeeded. */
|
|
114
|
+
private lastJudgement?;
|
|
115
|
+
constructor(task: string, options: AgentOptions);
|
|
116
|
+
/** The run history (populated as the loop runs). */
|
|
117
|
+
get history(): AgentHistoryList;
|
|
118
|
+
/** The effective (cloned, vision-filtered) registry the agent drives. */
|
|
119
|
+
get effectiveRegistry(): Registry;
|
|
120
|
+
/** The model this agent reasons with. */
|
|
121
|
+
get currentModel(): Model<Api>;
|
|
122
|
+
/** The GIF run-summary built after `run()`, when `output.generateGif` is on. */
|
|
123
|
+
get gif(): Uint8Array | undefined;
|
|
124
|
+
/** The HAR 1.2 network-traffic artifact built after `run()`, when `output.recordHar` is on. */
|
|
125
|
+
get har(): HarLog | undefined;
|
|
126
|
+
/** The video-as-GIF artifact built after `run()`, when `output.recordVideo` is on. */
|
|
127
|
+
get video(): Uint8Array | undefined;
|
|
128
|
+
/**
|
|
129
|
+
* The post-run judge's verdict, when `judge` is on. Informational — it never
|
|
130
|
+
* overrides the agent's own reported success. `undefined` if the judge was off
|
|
131
|
+
* or the evaluation failed.
|
|
132
|
+
*/
|
|
133
|
+
get judgement(): JudgementResult | undefined;
|
|
134
|
+
/** The curated O2 event stream as an async iterable (`for await`). */
|
|
135
|
+
events(): AsyncIterableIterator<AgentStreamEvent>;
|
|
136
|
+
/** Subscribe to one event type (callback sugar over {@link events}). */
|
|
137
|
+
on<T extends AgentStreamEventType>(type: T, cb: (event: AgentStreamEventOf<T>) => void): () => void;
|
|
138
|
+
/** Subscribe to every event. Returns an unsubscribe function. */
|
|
139
|
+
onEvent(cb: (event: AgentStreamEvent) => void): () => void;
|
|
140
|
+
run(options?: RunOptions): Promise<AgentHistoryList>;
|
|
141
|
+
/** Drive one step (respects pause/stop). Returns whether the task is now done. */
|
|
142
|
+
step(): Promise<boolean>;
|
|
143
|
+
pause(): void;
|
|
144
|
+
resume(): void;
|
|
145
|
+
stop(): void;
|
|
146
|
+
/** Clone the caller's (or default) registry and apply the vision filter (E7). */
|
|
147
|
+
private buildRegistry;
|
|
148
|
+
/**
|
|
149
|
+
* Compose the caller's `onDone` (if any) with the post-run artifact steps:
|
|
150
|
+
* the caller's hook always runs first, then — for whichever of GIF/HAR/video
|
|
151
|
+
* is gated on — the run's screenshots/network trace/screencast are turned
|
|
152
|
+
* into {@link Agent.gif} / {@link Agent.har} / {@link Agent.video}. Each
|
|
153
|
+
* artifact step is independently logged-and-swallowed on failure — a pure
|
|
154
|
+
* output artifact must never fail the run.
|
|
155
|
+
*/
|
|
156
|
+
private buildHooks;
|
|
157
|
+
/** Create the FileSystem + StepLoop once; apply any buffered pause/stop. */
|
|
158
|
+
private ensureReady;
|
|
159
|
+
}
|
|
160
|
+
//# sourceMappingURL=agent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/agent/agent.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,MAAM,EAAE,MAAM,eAAe,CAAC;AAGhF,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,yBAAyB,CAAC;AAEpE,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAgB,MAAM,mBAAmB,CAAC;AACpF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EAC1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,KAAK,UAAU,EAAkB,MAAM,OAAO,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,KAAK,eAAe,EAAY,MAAM,SAAS,CAAC;AAGzD,OAAO,EAAE,KAAK,UAAU,EAAY,KAAK,aAAa,EAAE,KAAK,cAAc,EAAE,MAAM,aAAa,CAAC;AACjG,OAAO,EAAE,KAAK,YAAY,EAAuB,MAAM,SAAS,CAAC;AAkBjE,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE;QACH,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAClB,gEAAgE;QAChE,UAAU,CAAC,EAAE,aAAa,CAAC;QAC3B;;;;;;;WAOG;QACH,aAAa,CAAC,EAAE,qBAAqB,CAAC;KACvC,CAAC;IACF,OAAO,EAAE;QACP,oCAAoC;QACpC,OAAO,EAAE,cAAc,CAAC;QACxB,gDAAgD;QAChD,MAAM,CAAC,EAAE,aAAa,CAAC;KACxB,CAAC;IACF,wEAAwE;IACxE,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,mDAAmD;IACnD,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,MAAM,CAAC,EAAE;QACP,mDAAmD;QACnD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,+DAA+D;QAC/D,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,mGAAmG;QACnG,WAAW,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;QACnC,yFAAyF;QACzF,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,mGAAmG;QACnG,WAAW,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;KACtC,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;IACjC,QAAQ,CAAC,EAAE;QACT,aAAa,CAAC,EAAE,aAAa,CAAC;QAC9B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC/B,CAAC;IACF,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,uEAAuE;IACvE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;;;;OASG;IACH,KAAK,CAAC,EACF,OAAO,GACP;QACE,8FAA8F;QAC9F,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACnB,0GAA0G;QAC1G,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gFAAgF;QAChF,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,6DAA6D;QAC7D,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,CAAC;CACP;AA8BD,qBAAa,KAAK;IAChB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAa;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA0B;IACtD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,OAAO,CAAC,UAAU,CAAC,CAAa;IAChC,OAAO,CAAC,QAAQ,CAAC,CAAW;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,cAAc,CAAS;IAC/B,0EAA0E;IAC1E,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,wEAAwE;IACxE,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,0EAA0E;IAC1E,OAAO,CAAC,SAAS,CAAC,CAAa;IAC/B,qFAAqF;IACrF,OAAO,CAAC,aAAa,CAAC,CAAkB;gBAE5B,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY;IA2B/C,oDAAoD;IACpD,IAAI,OAAO,IAAI,gBAAgB,CAE9B;IAED,yEAAyE;IACzE,IAAI,iBAAiB,IAAI,QAAQ,CAEhC;IAED,yCAAyC;IACzC,IAAI,YAAY,IAAI,KAAK,CAAC,GAAG,CAAC,CAE7B;IAED,gFAAgF;IAChF,IAAI,GAAG,IAAI,UAAU,GAAG,SAAS,CAEhC;IAED,+FAA+F;IAC/F,IAAI,GAAG,IAAI,MAAM,GAAG,SAAS,CAE5B;IAED,sFAAsF;IACtF,IAAI,KAAK,IAAI,UAAU,GAAG,SAAS,CAElC;IAED;;;;OAIG;IACH,IAAI,SAAS,IAAI,eAAe,GAAG,SAAS,CAE3C;IAED,sEAAsE;IACtE,MAAM,IAAI,qBAAqB,CAAC,gBAAgB,CAAC;IAIjD,wEAAwE;IACxE,EAAE,CAAC,CAAC,SAAS,oBAAoB,EAC/B,IAAI,EAAE,CAAC,EACP,EAAE,EAAE,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,KAAK,IAAI,GACzC,MAAM,IAAI;IAIb,iEAAiE;IACjE,OAAO,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,MAAM,IAAI;IAIpD,GAAG,CAAC,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAW9D,kFAAkF;IAC5E,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;IAK9B,KAAK,IAAI,IAAI;IAKb,MAAM,IAAI,IAAI;IAKd,IAAI,IAAI,IAAI;IAOZ,iFAAiF;IACjF,OAAO,CAAC,aAAa;IAWrB;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IAsElB,4EAA4E;YAC9D,WAAW;CA6B1B"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { AgentBackend, AgentEvent, AgentMessage, AgentRunOptions, AgentRunResult, BackendCapability } from "@ap3x/agent-core";
|
|
2
|
+
import type { Model } from "@ap3x/ai";
|
|
3
|
+
import { Agent, type AgentOptions } from "./agent";
|
|
4
|
+
import type { AgentStreamEvent } from "./event-stream";
|
|
5
|
+
/**
|
|
6
|
+
* Bridge one O2 {@link AgentStreamEvent} to an agent-core {@link AgentEvent}
|
|
7
|
+
* (undefined = intentionally dropped, no clean analog). Every one of the 11
|
|
8
|
+
* `AgentStreamEvent` variants is a named case below — 6 bridged, 5 dropped —
|
|
9
|
+
* so the drop set is explicit rather than a silent `default:` fall-through.
|
|
10
|
+
* `default` is reached only by a FUTURE variant the union doesn't have yet:
|
|
11
|
+
* with all 11 cases present, `event` is typed `never` there, so adding a 12th
|
|
12
|
+
* variant without a case above is a compile error (exhaustiveness guard).
|
|
13
|
+
* `backend.test.ts` pins the RUNTIME half — that each case still maps to the
|
|
14
|
+
* value asserted here, which the compiler alone can't guarantee.
|
|
15
|
+
*/
|
|
16
|
+
export declare function bridgeEvent(event: AgentStreamEvent): AgentEvent | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* Adapt an {@link Agent} (or {@link AgentOptions}) to agent-core's
|
|
19
|
+
* {@link AgentBackend}. An `Agent` instance is run as-is; `AgentOptions` spins a
|
|
20
|
+
* fresh ephemeral `Agent` per `run()` (the task arg supplies its task).
|
|
21
|
+
*/
|
|
22
|
+
export declare class BrowserAgentBackend implements AgentBackend {
|
|
23
|
+
private readonly source;
|
|
24
|
+
private readonly model;
|
|
25
|
+
private readonly listeners;
|
|
26
|
+
private active?;
|
|
27
|
+
constructor(source: Agent | AgentOptions);
|
|
28
|
+
private agentFor;
|
|
29
|
+
run(task: string | AgentMessage[], opts?: AgentRunOptions): Promise<AgentRunResult>;
|
|
30
|
+
/** Drain the O2 stream, translating each event to a subscribed agent-core listener. */
|
|
31
|
+
private pumpEvents;
|
|
32
|
+
private toRunResult;
|
|
33
|
+
subscribe(listener: (event: AgentEvent) => void | Promise<void>): () => void;
|
|
34
|
+
abort(): void;
|
|
35
|
+
supports(capability: BackendCapability): boolean;
|
|
36
|
+
getCurrentModel(): Model<string>;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/agent/backend.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,eAAe,EACf,cAAc,EACd,iBAAiB,EAClB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAO,KAAK,EAAwC,MAAM,UAAU,CAAC;AAEjF,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAwEvD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,UAAU,GAAG,SAAS,CA8C3E;AAED;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,YAAY;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAa;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA0D;IACpF,OAAO,CAAC,MAAM,CAAC,CAAQ;gBAEX,MAAM,EAAE,KAAK,GAAG,YAAY;IAKxC,OAAO,CAAC,QAAQ;IAIV,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,EAAE,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IAezF,uFAAuF;YACzE,UAAU;IAQxB,OAAO,CAAC,WAAW;IAcnB,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI;IAO5E,KAAK,IAAI,IAAI;IAIb,QAAQ,CAAC,UAAU,EAAE,iBAAiB,GAAG,OAAO;IAIhD,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;CAKjC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Known captcha/anti-bot vendors this heuristic recognizes; "generic" covers everything else. */
|
|
2
|
+
export type CaptchaVendor = "recaptcha" | "hcaptcha" | "cloudflare" | "perimeterx" | "datadome" | "akamai" | "generic";
|
|
3
|
+
export interface CaptchaSignal {
|
|
4
|
+
vendor: CaptchaVendor;
|
|
5
|
+
url: string;
|
|
6
|
+
title: string;
|
|
7
|
+
/** The text snippet that triggered detection (for logging/debugging). */
|
|
8
|
+
matchedText: string;
|
|
9
|
+
}
|
|
10
|
+
/** Page state the detector needs — a subset of `BrowserStateSummary`. */
|
|
11
|
+
export interface CaptchaDetectInput {
|
|
12
|
+
url: string;
|
|
13
|
+
title: string;
|
|
14
|
+
serializedText: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Detect a likely captcha/anti-bot interstitial from perceived page state.
|
|
18
|
+
* Heuristic, never a solver: returns a signal or `undefined`, nothing more.
|
|
19
|
+
*/
|
|
20
|
+
export declare function detectCaptcha(state: CaptchaDetectInput): CaptchaSignal | undefined;
|
|
21
|
+
/**
|
|
22
|
+
* Dedupe helper: fires only when the signal differs from the last one seen
|
|
23
|
+
* (by vendor + url) — keeps a host from being spammed once per step while the
|
|
24
|
+
* agent is stuck on the same captcha page.
|
|
25
|
+
*/
|
|
26
|
+
export declare class CaptchaWatchdog {
|
|
27
|
+
private last;
|
|
28
|
+
/** Check state; returns a signal only the FIRST time it's seen for this vendor+url. */
|
|
29
|
+
check(state: CaptchaDetectInput): CaptchaSignal | undefined;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=captcha-watchdog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"captcha-watchdog.d.ts","sourceRoot":"","sources":["../../src/agent/captcha-watchdog.ts"],"names":[],"mappings":"AAWA,kGAAkG;AAClG,MAAM,MAAM,aAAa,GACrB,WAAW,GACX,UAAU,GACV,YAAY,GACZ,YAAY,GACZ,UAAU,GACV,QAAQ,GACR,SAAS,CAAC;AAEd,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,aAAa,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,yEAAyE;AACzE,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB;AAmBD;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,kBAAkB,GAAG,aAAa,GAAG,SAAS,CAWlF;AAED;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,IAAI,CAAqB;IAEjC,uFAAuF;IACvF,KAAK,CAAC,KAAK,EAAE,kBAAkB,GAAG,aAAa,GAAG,SAAS;CAW5D"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { BrowserDriver, BrowserStateSummary } from "@ap3x/browser";
|
|
2
|
+
import type { AgentOutput } from "./output-schema";
|
|
3
|
+
/** How screenshots are attached across the run (E7). */
|
|
4
|
+
export type VisionMode = "off" | "everyStep" | "onDemand";
|
|
5
|
+
/** Read current browser state; capture a screenshot unless vision is off. */
|
|
6
|
+
export declare function perceive(driver: BrowserDriver, vision: VisionMode): Promise<BrowserStateSummary>;
|
|
7
|
+
/** Nudge when the run has consumed most of its step budget. */
|
|
8
|
+
export declare function budgetNudge(stepNumber: number, maxSteps: number): string | undefined;
|
|
9
|
+
/**
|
|
10
|
+
* Soft loop / stagnation detector. Never blocks an action — it only produces
|
|
11
|
+
* advisory nudge text. Tracks a rolling window of action signatures and the last
|
|
12
|
+
* few page fingerprints.
|
|
13
|
+
*/
|
|
14
|
+
export declare class LoopDetector {
|
|
15
|
+
private readonly actionHashes;
|
|
16
|
+
private readonly fingerprints;
|
|
17
|
+
private stagnation;
|
|
18
|
+
/** Record a step's executed actions and the page state it was taken against. */
|
|
19
|
+
record(output: AgentOutput | undefined, state: BrowserStateSummary): void;
|
|
20
|
+
/** Advisory nudges to inject this step (may be empty). */
|
|
21
|
+
nudges(): string[];
|
|
22
|
+
private maxRepetition;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=context-builder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-builder.d.ts","sourceRoot":"","sources":["../../src/agent/context-builder.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,wDAAwD;AACxD,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,GAAG,UAAU,CAAC;AAU1D,6EAA6E;AAC7E,wBAAgB,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAEhG;AAED,+DAA+D;AAC/D,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAIpF;AAcD;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,UAAU,CAAK;IAEvB,gFAAgF;IAChF,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAazE,0DAA0D;IAC1D,MAAM,IAAI,MAAM,EAAE;IAgBlB,OAAO,CAAC,aAAa;CAUtB"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { type Action, type ActionResult } from "../tools/types";
|
|
2
|
+
import type { CaptchaSignal } from "./captcha-watchdog";
|
|
3
|
+
import type { AgentOutput } from "./output-schema";
|
|
4
|
+
/** The discriminated union of everything a run emits to consumers. */
|
|
5
|
+
export type AgentStreamEvent = {
|
|
6
|
+
type: "run-started";
|
|
7
|
+
task: string;
|
|
8
|
+
maxSteps: number;
|
|
9
|
+
startedAt: number;
|
|
10
|
+
} | {
|
|
11
|
+
type: "step-started";
|
|
12
|
+
step: number;
|
|
13
|
+
url: string;
|
|
14
|
+
title: string;
|
|
15
|
+
} | {
|
|
16
|
+
type: "model-thought";
|
|
17
|
+
step: number;
|
|
18
|
+
thinking?: string;
|
|
19
|
+
evaluationPreviousGoal?: string;
|
|
20
|
+
memory?: string;
|
|
21
|
+
nextGoal?: string;
|
|
22
|
+
} | {
|
|
23
|
+
type: "action-started";
|
|
24
|
+
step: number;
|
|
25
|
+
/** Position of this action within the step's batch. */
|
|
26
|
+
index: number;
|
|
27
|
+
name: string;
|
|
28
|
+
/** The typed action (for live element highlighting etc.). */
|
|
29
|
+
action: Action;
|
|
30
|
+
params: Record<string, unknown>;
|
|
31
|
+
} | {
|
|
32
|
+
type: "action-finished";
|
|
33
|
+
step: number;
|
|
34
|
+
index: number;
|
|
35
|
+
name: string;
|
|
36
|
+
result: ActionResult;
|
|
37
|
+
} | {
|
|
38
|
+
type: "step-finished";
|
|
39
|
+
step: number;
|
|
40
|
+
/** The voice sentence — built from the model's own fields, no extra LLM cost. */
|
|
41
|
+
narration: string;
|
|
42
|
+
output?: AgentOutput;
|
|
43
|
+
results: ActionResult[];
|
|
44
|
+
} | {
|
|
45
|
+
type: "paused";
|
|
46
|
+
} | {
|
|
47
|
+
type: "resumed";
|
|
48
|
+
} | {
|
|
49
|
+
type: "done";
|
|
50
|
+
success: boolean;
|
|
51
|
+
finalText: string;
|
|
52
|
+
steps: number;
|
|
53
|
+
} | {
|
|
54
|
+
type: "error";
|
|
55
|
+
message: string;
|
|
56
|
+
step?: number;
|
|
57
|
+
} | ({
|
|
58
|
+
type: "captcha-detected";
|
|
59
|
+
step: number;
|
|
60
|
+
} & CaptchaSignal);
|
|
61
|
+
/** The `type` discriminant of {@link AgentStreamEvent}. */
|
|
62
|
+
export type AgentStreamEventType = AgentStreamEvent["type"];
|
|
63
|
+
/** The event variant for a given `type`. */
|
|
64
|
+
export type AgentStreamEventOf<T extends AgentStreamEventType> = Extract<AgentStreamEvent, {
|
|
65
|
+
type: T;
|
|
66
|
+
}>;
|
|
67
|
+
type AnyListener = (event: AgentStreamEvent) => void;
|
|
68
|
+
/**
|
|
69
|
+
* The producer + fan-out for {@link AgentStreamEvent}. The loop calls `emit`;
|
|
70
|
+
* consumers use `on` (typed callback), `onEvent` (every event) or `events()`
|
|
71
|
+
* (async iterable). Multiple consumers of each kind are supported; every one
|
|
72
|
+
* sees every event from the moment it subscribes.
|
|
73
|
+
*/
|
|
74
|
+
export declare class AgentEventStream {
|
|
75
|
+
private readonly typed;
|
|
76
|
+
private readonly any;
|
|
77
|
+
/** Fan an event out to all listeners. */
|
|
78
|
+
emit(event: AgentStreamEvent): void;
|
|
79
|
+
/** Subscribe to one event type. Returns an unsubscribe function. */
|
|
80
|
+
on<T extends AgentStreamEventType>(type: T, cb: (event: AgentStreamEventOf<T>) => void): () => void;
|
|
81
|
+
/** Subscribe to every event. Returns an unsubscribe function. */
|
|
82
|
+
onEvent(cb: AnyListener): () => void;
|
|
83
|
+
/**
|
|
84
|
+
* An async iterable of every event, ending after the terminal (done/error).
|
|
85
|
+
* Subscribe before/while a run produces events; iterating after a run has
|
|
86
|
+
* already finished (with no further run) parks on the first `next()`.
|
|
87
|
+
*/
|
|
88
|
+
events(): AsyncIterableIterator<AgentStreamEvent>;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Reconstruct the typed {@link Action} from a parsed single-key action. The
|
|
92
|
+
* built-in action schemas use the same field names as the {@link Action} union,
|
|
93
|
+
* so the discriminant is the action name and the params carry the rest.
|
|
94
|
+
*/
|
|
95
|
+
export declare function toTypedAction(name: string, params: Record<string, unknown>): Action;
|
|
96
|
+
/**
|
|
97
|
+
* The voice sentence for a step — assembled only from fields the model already
|
|
98
|
+
* produced (zero extra LLM cost). A terminal `done` speaks its own text; an
|
|
99
|
+
* ordinary step speaks its evaluation of the last goal plus the next goal,
|
|
100
|
+
* falling back to an action summary, then to the first error.
|
|
101
|
+
*/
|
|
102
|
+
export declare function assembleNarration(output: AgentOutput | undefined, results: readonly ActionResult[]): string;
|
|
103
|
+
export {};
|
|
104
|
+
//# sourceMappingURL=event-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event-stream.d.ts","sourceRoot":"","sources":["../../src/agent/event-stream.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAC9E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,sEAAsE;AACtE,MAAM,MAAM,gBAAgB,GACxB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1E;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAClE;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACD;IACE,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC,GACD;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,GAC5F;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB,GACD;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjD,CAAC;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CAAC,CAAC;AAEjE,2DAA2D;AAC3D,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;AAE5D,4CAA4C;AAC5C,MAAM,MAAM,kBAAkB,CAAC,CAAC,SAAS,oBAAoB,IAAI,OAAO,CACtE,gBAAgB,EAChB;IAAE,IAAI,EAAE,CAAC,CAAA;CAAE,CACZ,CAAC;AAOF,KAAK,WAAW,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;AAErD;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqD;IAC3E,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA0B;IAE9C,yCAAyC;IACzC,IAAI,CAAC,KAAK,EAAE,gBAAgB,GAAG,IAAI;IAKnC,oEAAoE;IACpE,EAAE,CAAC,CAAC,SAAS,oBAAoB,EAC/B,IAAI,EAAE,CAAC,EACP,EAAE,EAAE,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,KAAK,IAAI,GACzC,MAAM,IAAI;IAYb,iEAAiE;IACjE,OAAO,CAAC,EAAE,EAAE,WAAW,GAAG,MAAM,IAAI;IAOpC;;;;OAIG;IACH,MAAM,IAAI,qBAAqB,CAAC,gBAAgB,CAAC;CAsClD;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAEnF;AAUD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,SAAS,YAAY,EAAE,GAC/B,MAAM,CAsBR"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { BrowserSession } from "@ap3x/browser";
|
|
2
|
+
import type { ActionContext, Registry } from "../tools/registry";
|
|
3
|
+
import type { SecretMap, SensitiveData } from "../tools/sensitive";
|
|
4
|
+
import { type ActionResult } from "../tools/types";
|
|
5
|
+
import { type AgentStreamEvent } from "./event-stream";
|
|
6
|
+
export interface MultiActInput {
|
|
7
|
+
actions: Array<Record<string, unknown>>;
|
|
8
|
+
registry: Registry;
|
|
9
|
+
ctx: ActionContext;
|
|
10
|
+
session: BrowserSession;
|
|
11
|
+
secrets?: SecretMap;
|
|
12
|
+
/** Full cross-domain sensitive-data config, for outbound masking (M8 follow-up — see registry.execute). */
|
|
13
|
+
allSensitiveData?: SensitiveData;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
/**
|
|
16
|
+
* Checked BEFORE every action (design/06 §6.7, the stop/pause gate). Returns
|
|
17
|
+
* false → the batch breaks cleanly with the results so far. Lets a mid-step
|
|
18
|
+
* stop()/pause() interrupt the queued actions, not just an AbortSignal.
|
|
19
|
+
*/
|
|
20
|
+
shouldContinue?: () => boolean;
|
|
21
|
+
/** Cap on actions executed per step (default: no cap). */
|
|
22
|
+
maxActions?: number;
|
|
23
|
+
/** Highlight each action's element before it runs, cleared after (demo mode). */
|
|
24
|
+
demoMode?: boolean;
|
|
25
|
+
/** Emit the O2 action-started/action-finished events (optional). */
|
|
26
|
+
emit?: (event: AgentStreamEvent) => void;
|
|
27
|
+
/** 1-based step number, stamped onto emitted events. */
|
|
28
|
+
step?: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Execute a step's action batch. Returns the accumulated results (partial when the
|
|
32
|
+
* batch was aborted by a page change, an error, or a done). Never throws for
|
|
33
|
+
* action failures — those become `ActionResult.error`; it DOES throw if the run
|
|
34
|
+
* was aborted (the step loop maps that to a clean stop).
|
|
35
|
+
*/
|
|
36
|
+
export declare function multiAct(input: MultiActInput): Promise<ActionResult[]>;
|
|
37
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/agent/executor.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,KAAK,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AAC7E,OAAO,EAAE,KAAK,gBAAgB,EAAiB,MAAM,gBAAgB,CAAC;AAGtE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACxC,QAAQ,EAAE,QAAQ,CAAC;IACnB,GAAG,EAAE,aAAa,CAAC;IACnB,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,SAAS,CAAC;IACpB,2GAA2G;IAC3G,gBAAgB,CAAC,EAAE,aAAa,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,OAAO,CAAC;IAC/B,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,oEAAoE;IACpE,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACzC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AASD;;;;;GAKG;AACH,wBAAsB,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAsE5E"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { AgentHistoryList } from "./history";
|
|
2
|
+
import { type DecodedImage } from "./png-decode";
|
|
3
|
+
export interface GifOptions {
|
|
4
|
+
/** Milliseconds each frame is shown. Default 3000. */
|
|
5
|
+
frameDurationMs?: number;
|
|
6
|
+
/** Max palette colors per frame (default 256, GIF's ceiling). */
|
|
7
|
+
maxColors?: number;
|
|
8
|
+
/** GIF loop count: 0 = forever (default), -1 = play once, N = N extra loops. */
|
|
9
|
+
repeat?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Encode a list of same-size decoded frames into an animated GIF. Shared core
|
|
13
|
+
* used by {@link generateRunGif} (per-step screenshots) and the M11 video
|
|
14
|
+
* pipeline (agent/video.ts, screencast frames) — "assemble via the M10 GIF
|
|
15
|
+
* pipeline" per the PRP. Returns `undefined` when `frames` is empty.
|
|
16
|
+
*/
|
|
17
|
+
export declare function encodeFramesToGif(frames: readonly DecodedImage[], options?: GifOptions): Uint8Array | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Build an animated GIF from a run's history screenshots. Returns `undefined`
|
|
20
|
+
* when there is nothing to encode (no history, or no decodable screenshots).
|
|
21
|
+
*/
|
|
22
|
+
export declare function generateRunGif(history: AgentHistoryList, options?: GifOptions): Uint8Array | undefined;
|
|
23
|
+
//# sourceMappingURL=gif.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gif.d.ts","sourceRoot":"","sources":["../../src/agent/gif.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAgB,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,KAAK,YAAY,EAAmB,MAAM,cAAc,CAAC;AAclE,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,GAAE,UAAe,GACvB,UAAU,GAAG,SAAS,CAqBxB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,gBAAgB,EACzB,OAAO,GAAE,UAAe,GACvB,UAAU,GAAG,SAAS,CAExB"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { DOMInteractedElement, TabInfo } from "@ap3x/browser";
|
|
2
|
+
import { type ActionResult } from "../tools/types";
|
|
3
|
+
import type { UsageSummary } from "../usage/ledger";
|
|
4
|
+
import type { AgentOutput } from "./output-schema";
|
|
5
|
+
/** The slice of browser state persisted with a step. */
|
|
6
|
+
export interface BrowserStateHistory {
|
|
7
|
+
url: string;
|
|
8
|
+
title: string;
|
|
9
|
+
tabs: TabInfo[];
|
|
10
|
+
/** Present only when a screenshot was captured this step. */
|
|
11
|
+
screenshot?: string;
|
|
12
|
+
/**
|
|
13
|
+
* Per-action interacted-element snapshot, aligned index-for-index with
|
|
14
|
+
* `modelOutput.action` (undefined where an action carried no element index).
|
|
15
|
+
* Captured against the selector map the LLM actually saw (before any action
|
|
16
|
+
* ran this step) — design/04b §4.1 RECORD phase. This is the ONLY thing a
|
|
17
|
+
* replay (M9) has to re-locate an element across a fresh page load, since
|
|
18
|
+
* indices are live backendNodeIds that do not survive a reload.
|
|
19
|
+
*/
|
|
20
|
+
interactedElements?: Array<DOMInteractedElement | undefined>;
|
|
21
|
+
}
|
|
22
|
+
/** Per-step timing. `stepInterval` is the duration of the PREVIOUS step. */
|
|
23
|
+
export interface StepMetadata {
|
|
24
|
+
stepStartTime: number;
|
|
25
|
+
stepEndTime: number;
|
|
26
|
+
stepNumber: number;
|
|
27
|
+
stepInterval?: number;
|
|
28
|
+
}
|
|
29
|
+
/** Duration of one step in seconds. */
|
|
30
|
+
export declare function stepDurationSeconds(metadata: StepMetadata): number;
|
|
31
|
+
/** One recorded step. */
|
|
32
|
+
export interface AgentHistory {
|
|
33
|
+
modelOutput?: AgentOutput;
|
|
34
|
+
result: ActionResult[];
|
|
35
|
+
state: BrowserStateHistory;
|
|
36
|
+
metadata?: StepMetadata;
|
|
37
|
+
/** The raw state-message text, kept for debugging. */
|
|
38
|
+
stateMessage?: string;
|
|
39
|
+
}
|
|
40
|
+
/** Why the run loop stopped — a STRUCTURAL signal the loop sets, not a string sniff. */
|
|
41
|
+
export type TerminationReason = "done" | "max_steps" | "max_failures" | "run_timeout" | "aborted" | "stopped";
|
|
42
|
+
/** Append-only list of steps plus the run's usage summary + query helpers. */
|
|
43
|
+
export declare class AgentHistoryList {
|
|
44
|
+
readonly history: AgentHistory[];
|
|
45
|
+
usage?: UsageSummary;
|
|
46
|
+
/** Set once by the loop when it exits; drives StopReason mapping (no regex). */
|
|
47
|
+
terminationReason?: TerminationReason;
|
|
48
|
+
/** Append a step. Items are never mutated or reordered afterwards. */
|
|
49
|
+
add(item: AgentHistory): void;
|
|
50
|
+
get length(): number;
|
|
51
|
+
numberOfSteps(): number;
|
|
52
|
+
private lastResult;
|
|
53
|
+
/** The `extractedContent` of the last result of the last step. */
|
|
54
|
+
finalResult(): string | undefined;
|
|
55
|
+
/** True once a terminal `done` result has been recorded. */
|
|
56
|
+
isDone(): boolean;
|
|
57
|
+
/** The agent's self-reported success — only meaningful when done, else undefined. */
|
|
58
|
+
isSuccessful(): boolean | undefined;
|
|
59
|
+
/** One error (the first) per step; undefined where a step had none. */
|
|
60
|
+
errors(): Array<string | undefined>;
|
|
61
|
+
hasErrors(): boolean;
|
|
62
|
+
/** Per-step page URLs. */
|
|
63
|
+
urls(): string[];
|
|
64
|
+
/** Flattened action names across all steps (one per single-key action object). */
|
|
65
|
+
actionNames(): string[];
|
|
66
|
+
/** Sum of per-step durations (seconds). */
|
|
67
|
+
totalDurationSeconds(): number;
|
|
68
|
+
/** Plain-data snapshot for persistence (design/04b §4.1 `AgentHistoryList.save_to_file`). */
|
|
69
|
+
toJSON(): AgentHistoryFile;
|
|
70
|
+
/**
|
|
71
|
+
* Rebuild a list from a plain-data snapshot. The CALLER is responsible for
|
|
72
|
+
* schema-validating untrusted input before calling this (replay.ts's
|
|
73
|
+
* `loadHistoryFromFile` is the trust boundary — this constructor trusts its
|
|
74
|
+
* argument, matching every other in-process history mutation in this class).
|
|
75
|
+
*
|
|
76
|
+
* `interactedElements` entries round-trip through JSON as `null` (undefined
|
|
77
|
+
* array elements serialize that way — `Type.Array` has no other way to carry
|
|
78
|
+
* a "no element" slot); normalize them back to `undefined` here so in-memory
|
|
79
|
+
* histories built directly (never touching JSON) and loaded ones agree.
|
|
80
|
+
*/
|
|
81
|
+
static fromJSON(data: AgentHistoryFile): AgentHistoryList;
|
|
82
|
+
}
|
|
83
|
+
/** The serializable shape of an {@link AgentHistoryList} (M9 replay trust boundary). */
|
|
84
|
+
export interface AgentHistoryFile {
|
|
85
|
+
version: number;
|
|
86
|
+
history: AgentHistory[];
|
|
87
|
+
terminationReason?: string;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../src/agent/history.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACnE,OAAO,EAAE,KAAK,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,EAAE,CAAC;IAChB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,KAAK,CAAC,oBAAoB,GAAG,SAAS,CAAC,CAAC;CAC9D;AAED,4EAA4E;AAC5E,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,uCAAuC;AACvC,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,CAElE;AAED,yBAAyB;AACzB,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAUD,wFAAwF;AACxF,MAAM,MAAM,iBAAiB,GACzB,MAAM,GACN,WAAW,GACX,cAAc,GACd,aAAa,GACb,SAAS,GACT,SAAS,CAAC;AAEd,8EAA8E;AAC9E,qBAAa,gBAAgB;IAC3B,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAM;IACtC,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,gFAAgF;IAChF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IAEtC,sEAAsE;IACtE,GAAG,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI;IAI7B,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,aAAa,IAAI,MAAM;IAIvB,OAAO,CAAC,UAAU;IAMlB,kEAAkE;IAClE,WAAW,IAAI,MAAM,GAAG,SAAS;IAIjC,4DAA4D;IAC5D,MAAM,IAAI,OAAO;IAKjB,qFAAqF;IACrF,YAAY,IAAI,OAAO,GAAG,SAAS;IAMnC,uEAAuE;IACvE,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC;IAInC,SAAS,IAAI,OAAO;IAIpB,0BAA0B;IAC1B,IAAI,IAAI,MAAM,EAAE;IAIhB,kFAAkF;IAClF,WAAW,IAAI,MAAM,EAAE;IAWvB,2CAA2C;IAC3C,oBAAoB,IAAI,MAAM;IAO9B,6FAA6F;IAC7F,MAAM,IAAI,gBAAgB;IAI1B;;;;;;;;;;OAUG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,GAAG,gBAAgB;CAa1D;AAED,wFAAwF;AACxF,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|