handhold-sdk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +427 -0
- package/dist/d44f1fe8001e32f8e74d.svg +3056 -0
- package/dist/fonts/Phosphor.ttf +0 -0
- package/dist/fonts/Phosphor.woff +0 -0
- package/dist/fonts/Phosphor.woff2 +0 -0
- package/dist/handhold.min.js +1 -0
- package/package.json +114 -0
- package/src/HandHold.js +1599 -0
- package/src/agent/VENDOR.md +203 -0
- package/src/agent/agent/AgentLoop.js +366 -0
- package/src/agent/agent/prompts.js +163 -0
- package/src/agent/agent/tools.js +148 -0
- package/src/agent/controller.js +235 -0
- package/src/agent/dom/actions.js +456 -0
- package/src/agent/dom/domTree.js +1761 -0
- package/src/agent/dom/redact.js +98 -0
- package/src/agent/dom/serializer.js +332 -0
- package/src/agent/dom/utils.js +99 -0
- package/src/agent/index.js +169 -0
- package/src/agent/llm/connector.js +143 -0
- package/src/agent/package.json +3 -0
- package/src/agent/policy/PolicyGuard.js +172 -0
- package/src/agent/site/manifest.js +145 -0
- package/src/agent/ui/ChatWidget.js +219 -0
- package/src/agent-mode/AgentMode.js +429 -0
- package/src/api/APIClient.js +258 -0
- package/src/core/Config.js +207 -0
- package/src/core/UiState.js +83 -0
- package/src/core/defaults.js +20 -0
- package/src/events.js +59 -0
- package/src/index.js +77 -0
- package/src/intent/ActionVerbMap.js +324 -0
- package/src/intent/IntentExtractor.js +317 -0
- package/src/observers/ElementScanner.js +284 -0
- package/src/observers/NavigationObserver.js +182 -0
- package/src/observers/PageObserver.js +293 -0
- package/src/session/SessionManager.js +402 -0
- package/src/session/SessionStorage.js +148 -0
- package/src/ui/HelpWidget.js +1189 -0
- package/src/ui/UICoach.js +1725 -0
- package/src/ui/components/Button.css +137 -0
- package/src/ui/components/Button.js +102 -0
- package/src/ui/widget.css +599 -0
- package/src/utils/Logger.js +132 -0
- package/src/utils/MarkdownRenderer.js +39 -0
- package/src/utils/domUtils.js +350 -0
- package/src/utils/eventBus.js +102 -0
- package/src/utils/index.js +8 -0
- package/src/utils/stringUtils.js +202 -0
- package/types/index.d.ts +538 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# VENDOR — trusted-page-agent (Handhold action engine)
|
|
2
|
+
|
|
3
|
+
This directory (`sdk/src/agent/`) is a **vendored drop** of the security-audited
|
|
4
|
+
`trusted-page-agent` engine. It is the safe action layer for Handhold's
|
|
5
|
+
"Do it for me" mode. Treat the engine source as audited code: change it **only**
|
|
6
|
+
through documented patches recorded in this file, so future upstream syncs stay
|
|
7
|
+
possible.
|
|
8
|
+
|
|
9
|
+
## Origin
|
|
10
|
+
|
|
11
|
+
| Field | Value |
|
|
12
|
+
|---|---|
|
|
13
|
+
| Upstream package | `trusted-page-agent` |
|
|
14
|
+
| Version | `0.1.0` (frozen as the vendor drop / "P0" in the master plan) |
|
|
15
|
+
| Source in this repo | `handhold-agent-integration/vendor/trusted-page-agent/` |
|
|
16
|
+
| Upstream commit | none — delivered as a plain source drop (no `.git`); provenance is the security audit in the drop's `SECURITY.md` (alibaba/page-agent @ main, audited 2026-07-05) |
|
|
17
|
+
| Vendored on | 2026-07-06 (task H1) |
|
|
18
|
+
| Engine source checksum | `find . -name '*.js' -exec md5 -q {} \; | sort | md5` → `75cc24f480489f8ca90e37513372b961` |
|
|
19
|
+
|
|
20
|
+
The security guarantees this engine must preserve are documented in the drop's
|
|
21
|
+
`SECURITY.md`. The load-bearing ones for Handhold: no model-authored code ever
|
|
22
|
+
executes (no `eval` / `new Function` / script-tool); actions only on allowlisted
|
|
23
|
+
origins; sensitive actions fail closed on human confirm; PII redacted before
|
|
24
|
+
transmission; every decision audited.
|
|
25
|
+
|
|
26
|
+
## What was copied
|
|
27
|
+
|
|
28
|
+
- `handhold-agent-integration/vendor/trusted-page-agent/src/` → `sdk/src/agent/`
|
|
29
|
+
(domTree, serializer, redact, actions, utils, tools, prompts, AgentLoop,
|
|
30
|
+
controller, PolicyGuard, connector, ChatWidget, index).
|
|
31
|
+
- `handhold-agent-integration/vendor/trusted-page-agent/test/` → `sdk/test/agent/`.
|
|
32
|
+
|
|
33
|
+
At H1 the engine **source `.js` files were byte-for-byte identical** to the drop
|
|
34
|
+
(verified by matching md5 above). Since then the only engine-source change is
|
|
35
|
+
patch #5 below (`dom/serializer.js`, task H5); every other file remains
|
|
36
|
+
byte-identical to the drop. As of H5 the SDK imports the engine via
|
|
37
|
+
`sdk/src/agent-mode/AgentMode.js` (lazy `import()` bundled eagerly by webpack).
|
|
38
|
+
|
|
39
|
+
## Deviations from the drop (the full patch list)
|
|
40
|
+
|
|
41
|
+
Every change made while vendoring, so an upstream re-sync is a mechanical replay:
|
|
42
|
+
|
|
43
|
+
1. **Added `sdk/src/agent/package.json`** = `{"type":"module"}`.
|
|
44
|
+
Why: the engine is authored as ESM. The root SDK is CommonJS (webpack config
|
|
45
|
+
uses `require`), so we cannot set `"type":"module"` at the SDK root. A nested
|
|
46
|
+
package.json scopes ESM to this directory only — Node resolves module type
|
|
47
|
+
from the nearest package.json, and webpack/babel bundle ESM regardless.
|
|
48
|
+
**Not an engine-source change.**
|
|
49
|
+
|
|
50
|
+
2. **Added `sdk/test/agent/package.json`** = `{"type":"module"}`. Same reason,
|
|
51
|
+
so `node --test` treats the relocated tests as ESM.
|
|
52
|
+
|
|
53
|
+
3. **Rewrote test import paths** in `sdk/test/agent/*.test.js`:
|
|
54
|
+
`'../src/...'` → `'../../src/agent/...'`. Purely mechanical — tests moved from
|
|
55
|
+
`<drop>/test/` (sibling of `src/`) to `sdk/test/agent/` (two levels from
|
|
56
|
+
`sdk/src/agent/`). No test logic changed (verified by `diff`: only the
|
|
57
|
+
`import ... from` lines differ).
|
|
58
|
+
|
|
59
|
+
4. **Added `sdk/test/agent/invariants.test.js`** — a grep-invariant test that
|
|
60
|
+
fails if `eval(` / `new Function(` / `.innerHTML` / `document.write(` /
|
|
61
|
+
dynamic `import(` appears in executable code under `sdk/src/agent/` (comments
|
|
62
|
+
and JSDoc `import()` type annotations are stripped first). The drop shipped
|
|
63
|
+
this guarantee as prose in `SECURITY.md` but **not** as an automated check;
|
|
64
|
+
this test codifies it so the invariant "travels with the vendor drop" and
|
|
65
|
+
stays green in CI. Proven to have teeth (fails on an injected `eval`/`innerHTML`
|
|
66
|
+
probe). **Not an engine-source change** — it guards the engine, does not modify it.
|
|
67
|
+
|
|
68
|
+
5. **Patched `sdk/src/agent/dom/serializer.js`** (task H5, 2026-07-06 — first
|
|
69
|
+
engine-source change; marked `@handhold-patch H5` in the file):
|
|
70
|
+
- Extracted the inline blacklist/whitelist resolver in `getFlatTree` into an
|
|
71
|
+
exported `resolveElementList`, and extended it to FLATTEN function results
|
|
72
|
+
that are element collections (array/NodeList/any iterable), dropping
|
|
73
|
+
nullish results. Upstream mapped fn→value 1:1, which only supports
|
|
74
|
+
blacklisting exact elements (`domTree.js` matches `includes(element)`).
|
|
75
|
+
- Why: Handhold's `AgentMode` must hide the ENTIRE widget subtree
|
|
76
|
+
(`#handhold-container` and every descendant) from engine snapshots, so the
|
|
77
|
+
agent can never see or act on Handhold's own UI (e.g. click its own Allow
|
|
78
|
+
button). The entry is a function re-evaluated on every snapshot, so
|
|
79
|
+
dynamically rendered widget UI (confirm cards, activity lines) stays
|
|
80
|
+
excluded.
|
|
81
|
+
- Diff (semantic): `const resolve = (list=[]) => list.map(fn?fn():item)` →
|
|
82
|
+
`export const resolveElementList = (list=[]) => list.flatMap(...)` with
|
|
83
|
+
iterable-spread + nullish-drop; `getFlatTree` now uses it. No other engine
|
|
84
|
+
file touched; behavior for existing element/function-returning-element
|
|
85
|
+
entries is unchanged.
|
|
86
|
+
- Tests: `sdk/test/agent/resolve-blacklist.test.js` (Handhold addition, 3
|
|
87
|
+
tests). Security-neutral-or-better: the change only ever REMOVES elements
|
|
88
|
+
from the agent's view.
|
|
89
|
+
|
|
90
|
+
6. **Added `sdk/src/agent/site/manifest.js`** (task H6.1, new module — intentional
|
|
91
|
+
forward-of-upstream addition): loadSiteManifest (THROWS unless
|
|
92
|
+
`reviewed === true`; refuses non-same-origin-absolute paths with an error
|
|
93
|
+
list), matchManifestPath (`:param` segments; unlisted → null),
|
|
94
|
+
listManifestPaths, describeCurrentPage. Tests:
|
|
95
|
+
`sdk/test/agent/site-manifest.test.js` (6).
|
|
96
|
+
|
|
97
|
+
7. **Patched `sdk/src/agent/agent/tools.js`** (task H6.2, `@handhold-patch H6`):
|
|
98
|
+
added the `navigate` tool (`{path}` required string, shape-only validation —
|
|
99
|
+
manifest membership + sensitivity enforced by the controller/guard). The
|
|
100
|
+
vendored exact-tool-set pin in `sdk/test/agent/agent.test.js` failed on the
|
|
101
|
+
addition (by design) and was consciously updated to include `navigate`
|
|
102
|
+
(marker comment in the test); it remains an exact pin and still asserts no
|
|
103
|
+
`execute_javascript`/`evaluate`. Tests:
|
|
104
|
+
`sdk/test/agent/navigate-tool.test.js` (4).
|
|
105
|
+
|
|
106
|
+
8. **Patched `sdk/src/agent/policy/PolicyGuard.js`** (task H6.3,
|
|
107
|
+
`@handhold-patch H6`): `check()` input gains optional `forceSensitive`
|
|
108
|
+
(string reason) — callers that KNOW an action is sensitive (manifest
|
|
109
|
+
`sensitive: true` pages) force the fail-closed confirm gate. Strictly
|
|
110
|
+
additive: it can only ADD the gate; keyword/attribute detection is unchanged.
|
|
111
|
+
|
|
112
|
+
9. **Patched `sdk/src/agent/controller.js` + `sdk/src/agent/index.js`**
|
|
113
|
+
(task H6.3, `@handhold-patch H6`): GuardedController accepts `siteManifest`
|
|
114
|
+
(validated) + injectable `navigateImpl` (default `location.assign`);
|
|
115
|
+
`act('navigate')` runs before the snapshot requirement — no manifest →
|
|
116
|
+
unavailable; unlisted path → refused BEFORE the guard with the valid-path
|
|
117
|
+
list (no audit/rate cost); listed path → PolicyGuard.check (origin re-check,
|
|
118
|
+
rate limit, audit; sensitive forces confirm) → `navigateImpl(origin+path)`.
|
|
119
|
+
TrustedPageAgent accepts raw `siteManifest` JSON and validates it AT
|
|
120
|
+
CONSTRUCTION via loadSiteManifest (unreviewed → throw), passes `navigateImpl`
|
|
121
|
+
through, and hands the validated manifest to the loop. Tests:
|
|
122
|
+
`sdk/test/agent/navigate-guarded.test.js` (9; sensitive-gate teeth proven).
|
|
123
|
+
|
|
124
|
+
10. **Patched `sdk/src/agent/agent/prompts.js` + `sdk/src/agent/agent/AgentLoop.js`**
|
|
125
|
+
(task H6.4, `@handhold-patch H6`): `buildSystemPrompt({manifest, task})`
|
|
126
|
+
renders a `<site_map>` section (trusted, human-reviewed — outside
|
|
127
|
+
`<untrusted_page_content>`) with SENSITIVE flags and reach trails, capped
|
|
128
|
+
~2 kB with task-keyword priority; `buildStateMessage` gains an optional
|
|
129
|
+
`Current page:` line; AgentLoop accepts `siteManifest`, builds the system
|
|
130
|
+
prompt per run with the task, resolves Current page via describeCurrentPage.
|
|
131
|
+
Zero-arg `buildSystemPrompt()` unchanged. Tests:
|
|
132
|
+
`sdk/test/agent/prompts-sitemap.test.js` (5).
|
|
133
|
+
|
|
134
|
+
11. **Patched `sdk/src/agent/agent/AgentLoop.js` + `sdk/src/agent/index.js`**
|
|
135
|
+
(task H7, `@handhold-patch H7`): checkpoint/resume support.
|
|
136
|
+
`AgentLoop.run(task, {restoredHistory, stepsUsed})` — restored entries are
|
|
137
|
+
prior CONTEXT, each prefixed `(restored)` (never double-prefixed) so the
|
|
138
|
+
model cannot read them as fresh instructions; the step budget CONTINUES
|
|
139
|
+
(first live step = stepsUsed + 1 of the same maxSteps). New
|
|
140
|
+
`getRunState()` → `{history, stepsUsed}` (loop) and TrustedPageAgent
|
|
141
|
+
proxy adding the guard's `audit` — read by the Handhold bridge immediately
|
|
142
|
+
before a navigation to write its checkpoint. Upstream activity/history
|
|
143
|
+
ordering deliberately untouched (the `executed` event still fires before
|
|
144
|
+
the history push; the bridge adds its own synthetic marker for the
|
|
145
|
+
in-flight navigation). Tests: `sdk/test/agent/resume-loop.test.js` (4;
|
|
146
|
+
prefix teeth proven).
|
|
147
|
+
|
|
148
|
+
12. **Patched `sdk/src/agent/agent/AgentLoop.js`** (task H9, `@handhold-patch H9`):
|
|
149
|
+
consecutive-repeat guard. Live testing surfaced runs that completed the task
|
|
150
|
+
but never called `done`, then repeated the same action toward the step cap
|
|
151
|
+
("continued endlessly"). The loop now tracks the `(actionName + JSON(input))`
|
|
152
|
+
signature: 3 identical repeats in a row inject a strong "call done or change
|
|
153
|
+
approach" nudge into history instead of executing; 5 stop the run with a
|
|
154
|
+
failure report. Any different action resets the counter, so legitimate
|
|
155
|
+
varied multi-step runs (and the 90 prior tests) are untouched. Reset per
|
|
156
|
+
run alongside history/step state. Tests:
|
|
157
|
+
`sdk/test/agent/repeat-guard.test.js` (3). (Complements Handhold-side task
|
|
158
|
+
framing in `AgentMode.buildTask` — done-criterion + ask-don't-invent — which
|
|
159
|
+
is not a vendor change.)
|
|
160
|
+
|
|
161
|
+
12b. **Patched `sdk/src/agent/agent/AgentLoop.js` + `agent/prompts.js`** (round 6,
|
|
162
|
+
`@handhold-patch H9`): action-format loop fix. Live, llama-3.3-70b took the
|
|
163
|
+
prompt's `tool_name` placeholder literally and emitted a FLAT action
|
|
164
|
+
(`{"tool_name":"click_element_by_index","index":3}`) instead of
|
|
165
|
+
tool-name-as-key (`{"click_element_by_index":{...}}`); the strict single-key
|
|
166
|
+
parser threw every step → infinite parse-fail loop (the repeat guard misses
|
|
167
|
+
it because parse failures `continue` before it). Three parts:
|
|
168
|
+
(1) `parseAgentOutput` tolerates flat variants (`tool_name`/`tool`/`name`/
|
|
169
|
+
`action_name` + inline fields or `input`/`args`/`arguments`/`parameters`);
|
|
170
|
+
security unchanged (validateToolInput + PolicyGuard still gate).
|
|
171
|
+
(2) prompt output-contract shows a concrete `{"click_element_by_index":
|
|
172
|
+
{"index":0}}` example and says "the tool name IS the key; do not use a
|
|
173
|
+
tool_name field."
|
|
174
|
+
(3) consecutive parse-failures are bounded (5 → stop, ≥3 → nudge with an
|
|
175
|
+
example), so a stuck model can't burn all maxSteps. Tests:
|
|
176
|
+
`sdk/test/agent/parse-tolerant.test.js` (6) + a parse-fail-loop case in
|
|
177
|
+
`repeat-guard.test.js`.
|
|
178
|
+
(5) consecutive LLM-call failures are bounded (>=3 → stop): when the provider
|
|
179
|
+
is down/rate-limited (Groq daily-token 429 → proxy error) every call throws;
|
|
180
|
+
without this the loop "thinks" through all maxSteps. Test in
|
|
181
|
+
`repeat-guard.test.js`.
|
|
182
|
+
(4) ask_user is bounded by PROGRESS, not a fixed total (a form can have any
|
|
183
|
+
number of fields): consecutive asks with no intervening successful page
|
|
184
|
+
action are capped (>4 → stop); any successful click/type/select/nav resets
|
|
185
|
+
the counter, so ask-then-fill for a 9-field form completes. The answer also
|
|
186
|
+
feeds back "use this EXACT value; do not ask again". Live: a model asked for
|
|
187
|
+
a title, got it, then asked forever (repeat guard misses it when wording
|
|
188
|
+
varies). Tests: loop + any-size-form cases in `repeat-guard.test.js`.
|
|
189
|
+
|
|
190
|
+
If a future task must patch engine source, add a numbered entry here with the
|
|
191
|
+
file, the diff, and the reason.
|
|
192
|
+
|
|
193
|
+
## Running the engine tests
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
cd sdk
|
|
197
|
+
npm run test:agent # node --test test/agent/*.test.js
|
|
198
|
+
# → 94 pass (62 engine + 1 invariant + 3 blacklist-resolve
|
|
199
|
+
# + 24 whole-site + 4 checkpoint/resume)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`npm test` (jest) is configured to ignore `test/agent/` — the engine suite runs
|
|
203
|
+
under `node --test`, not jest.
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AgentLoop.js — the ReAct loop (observe → think → act).
|
|
3
|
+
* Structure adapted from alibaba/page-agent PageAgentCore (MIT), rewritten:
|
|
4
|
+
* - every action goes through controller.act(), which enforces PolicyGuard;
|
|
5
|
+
* the loop itself has NO direct DOM access and no way to bypass the guard
|
|
6
|
+
* - LLM output is parsed as a JSON "macro action" (works with hosted models
|
|
7
|
+
* and local models that lack native tool calling)
|
|
8
|
+
* - unknown tools / invalid inputs never execute; the error is fed back to
|
|
9
|
+
* the model as an observation instead
|
|
10
|
+
*/
|
|
11
|
+
import { buildStateMessage, buildSystemPrompt } from './prompts.js'
|
|
12
|
+
import { validateToolInput } from './tools.js'
|
|
13
|
+
// @handhold-patch H6 (VENDOR.md): Current-page resolution from the site manifest.
|
|
14
|
+
import { describeCurrentPage } from '../site/manifest.js'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Extract the agent's JSON macro action from raw model text.
|
|
18
|
+
* Tolerates markdown fences and preamble/postamble chatter (common with
|
|
19
|
+
* small local models). Throws with a clear message when no JSON is found.
|
|
20
|
+
*/
|
|
21
|
+
export function parseAgentOutput(text) {
|
|
22
|
+
let candidate = null
|
|
23
|
+
|
|
24
|
+
// Prefer fenced ```json blocks if present.
|
|
25
|
+
const fence = text.match(/```(?:json)?\s*([\s\S]*?)```/)
|
|
26
|
+
if (fence) candidate = fence[1]
|
|
27
|
+
|
|
28
|
+
if (!candidate) {
|
|
29
|
+
// Find the first balanced top-level {...}
|
|
30
|
+
const start = text.indexOf('{')
|
|
31
|
+
if (start !== -1) {
|
|
32
|
+
let depth = 0
|
|
33
|
+
let inString = false
|
|
34
|
+
let escape = false
|
|
35
|
+
for (let i = start; i < text.length; i++) {
|
|
36
|
+
const ch = text[i]
|
|
37
|
+
if (escape) { escape = false; continue }
|
|
38
|
+
if (ch === '\\') { escape = true; continue }
|
|
39
|
+
if (ch === '"') inString = !inString
|
|
40
|
+
if (inString) continue
|
|
41
|
+
if (ch === '{') depth++
|
|
42
|
+
if (ch === '}') {
|
|
43
|
+
depth--
|
|
44
|
+
if (depth === 0) {
|
|
45
|
+
candidate = text.slice(start, i + 1)
|
|
46
|
+
break
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (!candidate) throw new Error('Model output contains no JSON object.')
|
|
54
|
+
|
|
55
|
+
let parsed
|
|
56
|
+
try {
|
|
57
|
+
parsed = JSON.parse(candidate)
|
|
58
|
+
} catch (e) {
|
|
59
|
+
throw new Error(`Model output JSON failed to parse: ${e.message}`)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const action = parsed.action
|
|
63
|
+
if (!action || typeof action !== 'object' || Array.isArray(action)) {
|
|
64
|
+
throw new Error('Model output is missing an "action" object.')
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* @handhold-patch H9 (VENDOR.md) — tolerant action extraction.
|
|
69
|
+
* Expected form is tool-name-as-key: { "click_element_by_index": {..} }.
|
|
70
|
+
* But smaller models (llama-3.3-70b) took the prompt's `tool_name` placeholder
|
|
71
|
+
* literally and emitted a FLAT object: { "tool_name": "click_...", "index": 3 },
|
|
72
|
+
* which the strict single-key rule rejected -> infinite parse-fail loop.
|
|
73
|
+
* Accept the common flat variants. Security is unchanged: validateToolInput
|
|
74
|
+
* strips unknown fields and rejects unknown tools, and PolicyGuard still gates.
|
|
75
|
+
*/
|
|
76
|
+
const NAME_KEYS = ['tool_name', 'tool', 'name', 'action_name']
|
|
77
|
+
const INPUT_KEYS = ['input', 'args', 'arguments', 'parameters', 'params']
|
|
78
|
+
let actionName, actionInput
|
|
79
|
+
const nameKey = NAME_KEYS.find((k) => typeof action[k] === 'string')
|
|
80
|
+
if (nameKey) {
|
|
81
|
+
actionName = action[nameKey]
|
|
82
|
+
const inputKey = INPUT_KEYS.find((k) => action[k] && typeof action[k] === 'object' && !Array.isArray(action[k]))
|
|
83
|
+
if (inputKey) {
|
|
84
|
+
actionInput = action[inputKey]
|
|
85
|
+
} else {
|
|
86
|
+
actionInput = { ...action }
|
|
87
|
+
delete actionInput[nameKey]
|
|
88
|
+
}
|
|
89
|
+
} else {
|
|
90
|
+
const names = Object.keys(action)
|
|
91
|
+
if (names.length !== 1) {
|
|
92
|
+
throw new Error(`"action" must name exactly one tool (as the key, e.g. {"click_element_by_index":{...}}), got keys: ${names.join(', ')}.`)
|
|
93
|
+
}
|
|
94
|
+
actionName = names[0]
|
|
95
|
+
actionInput = action[names[0]] ?? {}
|
|
96
|
+
}
|
|
97
|
+
if (typeof actionInput !== 'object' || actionInput === null || Array.isArray(actionInput)) {
|
|
98
|
+
actionInput = {}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return {
|
|
102
|
+
evaluation: String(parsed.evaluation_previous_goal ?? ''),
|
|
103
|
+
memory: String(parsed.memory ?? ''),
|
|
104
|
+
nextGoal: String(parsed.next_goal ?? ''),
|
|
105
|
+
actionName: String(actionName),
|
|
106
|
+
actionInput,
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export class AgentLoop {
|
|
111
|
+
#llm
|
|
112
|
+
#controller
|
|
113
|
+
#maxSteps
|
|
114
|
+
#onAskUser
|
|
115
|
+
#onActivity
|
|
116
|
+
#waitImpl
|
|
117
|
+
#siteManifest
|
|
118
|
+
#history = []
|
|
119
|
+
#stopped = false
|
|
120
|
+
#currentStep = 0
|
|
121
|
+
// @handhold-patch H9: consecutive-repeat guard (see VENDOR.md)
|
|
122
|
+
#lastSignature = null
|
|
123
|
+
#repeatCount = 0
|
|
124
|
+
#parseFailCount = 0
|
|
125
|
+
#asksSinceProgress = 0
|
|
126
|
+
#llmFailCount = 0
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* @param {Object} cfg
|
|
130
|
+
* @param {{complete: Function}} cfg.llm from createLLM()
|
|
131
|
+
* @param {{getBrowserState: Function, act: Function}} cfg.controller
|
|
132
|
+
* @param {number} [cfg.maxSteps=25]
|
|
133
|
+
* @param {(q:string)=>Promise<string>} [cfg.onAskUser]
|
|
134
|
+
* @param {(ev:Object)=>void} [cfg.onActivity] UI feedback hook
|
|
135
|
+
* @param {(s:number)=>Promise<void>} [cfg.waitImpl] injectable for tests
|
|
136
|
+
* @param {Object} [cfg.siteManifest] @handhold-patch H6: validated manifest —
|
|
137
|
+
* feeds the <site_map> system section + Current-page state line
|
|
138
|
+
*/
|
|
139
|
+
constructor(cfg) {
|
|
140
|
+
this.#llm = cfg.llm
|
|
141
|
+
this.#controller = cfg.controller
|
|
142
|
+
this.#maxSteps = cfg.maxSteps ?? 25
|
|
143
|
+
this.#onAskUser = cfg.onAskUser ?? null
|
|
144
|
+
this.#onActivity = cfg.onActivity ?? (() => {})
|
|
145
|
+
this.#waitImpl = cfg.waitImpl ?? ((s) => new Promise((r) => setTimeout(r, s * 1000)))
|
|
146
|
+
this.#siteManifest = cfg.siteManifest ?? null
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
historyText() {
|
|
150
|
+
return this.#history.join('\n')
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
stop() {
|
|
154
|
+
this.#stopped = true
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* @handhold-patch H7 (see sdk/src/agent/VENDOR.md) — checkpoint support.
|
|
159
|
+
* Snapshot of the run for cross-page persistence: the bridge reads this
|
|
160
|
+
* immediately before a navigation to write its checkpoint.
|
|
161
|
+
*/
|
|
162
|
+
getRunState() {
|
|
163
|
+
return { history: [...this.#history], stepsUsed: this.#currentStep }
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Run a task to completion.
|
|
168
|
+
* @handhold-patch H7: `restoredHistory` + `stepsUsed` resume a checkpointed
|
|
169
|
+
* run. Restored entries are PRIOR CONTEXT, never fresh instructions — each
|
|
170
|
+
* is prefixed "(restored)" (no double prefixing) — and the step budget
|
|
171
|
+
* CONTINUES: the first live step is stepsUsed + 1 of the same maxSteps.
|
|
172
|
+
* @returns {Promise<{success: boolean, text: string, steps: number}>}
|
|
173
|
+
*/
|
|
174
|
+
async run(task, { signal, restoredHistory, stepsUsed } = {}) {
|
|
175
|
+
this.#history = []
|
|
176
|
+
this.#stopped = false
|
|
177
|
+
this.#currentStep = 0
|
|
178
|
+
this.#lastSignature = null
|
|
179
|
+
this.#repeatCount = 0
|
|
180
|
+
this.#parseFailCount = 0
|
|
181
|
+
this.#asksSinceProgress = 0
|
|
182
|
+
this.#llmFailCount = 0
|
|
183
|
+
let startStep = 1
|
|
184
|
+
if (Array.isArray(restoredHistory) && restoredHistory.length) {
|
|
185
|
+
this.#history = restoredHistory.map((h) => {
|
|
186
|
+
const entry = String(h)
|
|
187
|
+
return entry.startsWith('(restored)') ? entry : `(restored) ${entry}`
|
|
188
|
+
})
|
|
189
|
+
if (Number.isInteger(stepsUsed) && stepsUsed > 0) {
|
|
190
|
+
startStep = Math.min(stepsUsed + 1, this.#maxSteps)
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const system = buildSystemPrompt({ manifest: this.#siteManifest, task })
|
|
194
|
+
|
|
195
|
+
for (let step = startStep; step <= this.#maxSteps; step++) {
|
|
196
|
+
this.#currentStep = step
|
|
197
|
+
if (this.#stopped || signal?.aborted) {
|
|
198
|
+
return { success: false, text: 'Stopped by user.', steps: step - 1 }
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// ---- observe ----
|
|
202
|
+
const state = await this.#controller.getBrowserState()
|
|
203
|
+
|
|
204
|
+
// ---- think ----
|
|
205
|
+
this.#onActivity({ type: 'thinking', step })
|
|
206
|
+
const userMessage = buildStateMessage({
|
|
207
|
+
task,
|
|
208
|
+
stepNumber: step,
|
|
209
|
+
maxSteps: this.#maxSteps,
|
|
210
|
+
historyText: this.historyText(),
|
|
211
|
+
url: state.url,
|
|
212
|
+
elementsString: state.elementsString,
|
|
213
|
+
currentPage: this.#siteManifest ? describeCurrentPage(this.#siteManifest, state.url) : null,
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
let reply
|
|
217
|
+
try {
|
|
218
|
+
reply = await this.#llm.complete({
|
|
219
|
+
system,
|
|
220
|
+
messages: [{ role: 'user', content: userMessage }],
|
|
221
|
+
signal,
|
|
222
|
+
})
|
|
223
|
+
this.#llmFailCount = 0
|
|
224
|
+
} catch (e) {
|
|
225
|
+
if (signal?.aborted) return { success: false, text: 'Stopped by user.', steps: step }
|
|
226
|
+
// @handhold-patch H9 (VENDOR.md): bound consecutive LLM failures.
|
|
227
|
+
// When the assistant service is down or rate-limited (e.g. Groq
|
|
228
|
+
// daily-token 429 -> proxy error), every call fails; without this
|
|
229
|
+
// the loop "thinks" its way through all maxSteps. Stop after 3.
|
|
230
|
+
this.#llmFailCount += 1
|
|
231
|
+
if (this.#llmFailCount >= 3) {
|
|
232
|
+
const text = `Stopped: the assistant service is unavailable right now (${e.message}). Please try again later.`
|
|
233
|
+
this.#onActivity({ type: 'done', step, success: false })
|
|
234
|
+
return { success: false, text, steps: step }
|
|
235
|
+
}
|
|
236
|
+
this.#history.push(`<step_${step}> LLM call failed: ${e.message}`)
|
|
237
|
+
continue
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
let parsed
|
|
241
|
+
try {
|
|
242
|
+
parsed = parseAgentOutput(reply.text)
|
|
243
|
+
this.#parseFailCount = 0
|
|
244
|
+
} catch (e) {
|
|
245
|
+
// @handhold-patch H9 (VENDOR.md): bound parse-fail loops. Parse
|
|
246
|
+
// failures `continue` before the repeat guard, so a model stuck on
|
|
247
|
+
// a bad output shape would otherwise burn all maxSteps. Stop after
|
|
248
|
+
// 5 in a row; nudge harder (with a concrete example) from the 3rd.
|
|
249
|
+
this.#parseFailCount += 1
|
|
250
|
+
if (this.#parseFailCount >= 5) {
|
|
251
|
+
const text = `Stopped: the model produced unparseable output ${this.#parseFailCount} times in a row (${e.message}).`
|
|
252
|
+
this.#onActivity({ type: 'done', step, success: false })
|
|
253
|
+
return { success: false, text, steps: step }
|
|
254
|
+
}
|
|
255
|
+
const example =
|
|
256
|
+
this.#parseFailCount >= 3
|
|
257
|
+
? ' A VALID reply looks like {"evaluation_previous_goal":"...","memory":"...","next_goal":"...","action":{"click_element_by_index":{"index":0}}} — the tool name is the KEY inside "action"; do NOT use a "tool_name" field.'
|
|
258
|
+
: ''
|
|
259
|
+
this.#history.push(
|
|
260
|
+
`<step_${step}> Your previous reply could not be parsed (${e.message}). Reply with ONE valid JSON object only.${example}`
|
|
261
|
+
)
|
|
262
|
+
continue
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ---- act ----
|
|
266
|
+
const { actionName, actionInput, nextGoal, evaluation, memory } = parsed
|
|
267
|
+
const valid = validateToolInput(actionName, actionInput)
|
|
268
|
+
if (!valid.ok) {
|
|
269
|
+
this.#history.push(
|
|
270
|
+
`<step_${step}> Goal: ${nextGoal} | Action ${actionName} REJECTED: ${valid.error}`
|
|
271
|
+
)
|
|
272
|
+
continue
|
|
273
|
+
}
|
|
274
|
+
const input = valid.value
|
|
275
|
+
|
|
276
|
+
if (actionName === 'done') {
|
|
277
|
+
this.#onActivity({ type: 'executing', step, actionName, input, nextGoal })
|
|
278
|
+
this.#history.push(`<step_${step}> done: ${input.text}`)
|
|
279
|
+
this.#onActivity({ type: 'done', step, success: input.success })
|
|
280
|
+
return { success: input.success !== false, text: input.text, steps: step }
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* @handhold-patch H9 (VENDOR.md) — consecutive-repeat guard.
|
|
285
|
+
* Live testing surfaced runs that completed the task but never called
|
|
286
|
+
* done, then repeated the same action toward the step cap. Track the
|
|
287
|
+
* (action + input) signature: after 3 identical repeats in a row, feed
|
|
288
|
+
* a strong nudge instead of executing; after 5, stop with a clear
|
|
289
|
+
* failure so the loop can never run away. Progress (any different
|
|
290
|
+
* action) resets the counter, so legitimate varied steps are untouched.
|
|
291
|
+
*/
|
|
292
|
+
const signature = `${actionName}:${JSON.stringify(input)}`
|
|
293
|
+
if (signature === this.#lastSignature) {
|
|
294
|
+
this.#repeatCount += 1
|
|
295
|
+
} else {
|
|
296
|
+
this.#lastSignature = signature
|
|
297
|
+
this.#repeatCount = 1
|
|
298
|
+
}
|
|
299
|
+
if (this.#repeatCount >= 5) {
|
|
300
|
+
const text = `Stopped: repeated the same action (${actionName}) ${this.#repeatCount} times with no progress. If the task looks complete it likely is; otherwise it is blocked.`
|
|
301
|
+
this.#onActivity({ type: 'done', step, success: false })
|
|
302
|
+
return { success: false, text, steps: step }
|
|
303
|
+
}
|
|
304
|
+
if (this.#repeatCount >= 3) {
|
|
305
|
+
this.#history.push(
|
|
306
|
+
`<step_${step}> You have chosen the SAME action ${this.#repeatCount} times in a row with no change. ` +
|
|
307
|
+
`If the objective is already achieved, call done(success=true) NOW. Otherwise try a DIFFERENT approach — do not repeat this action.`
|
|
308
|
+
)
|
|
309
|
+
continue
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
this.#onActivity({ type: 'executing', step, actionName, input, nextGoal })
|
|
313
|
+
|
|
314
|
+
let resultMessage
|
|
315
|
+
if (actionName === 'wait') {
|
|
316
|
+
await this.#waitImpl(input.seconds ?? 1)
|
|
317
|
+
resultMessage = `Waited ${input.seconds ?? 1}s.`
|
|
318
|
+
} else if (actionName === 'ask_user') {
|
|
319
|
+
// @handhold-patch H9 (VENDOR.md): bound ask_user WITHOUT a fixed
|
|
320
|
+
// total cap — a form can have any number of fields, so asking once
|
|
321
|
+
// per field (and filling it) must be unlimited. What's a loop is
|
|
322
|
+
// asking repeatedly WITHOUT making progress on the page. This
|
|
323
|
+
// counter resets on any successful page action (below); only
|
|
324
|
+
// consecutive asks with no intervening progress trip it.
|
|
325
|
+
this.#asksSinceProgress += 1
|
|
326
|
+
if (this.#asksSinceProgress > 4) {
|
|
327
|
+
const text = 'Stopped: I asked for input several times without being able to act on the answers. Please try again or do this step yourself.'
|
|
328
|
+
this.#onActivity({ type: 'done', step, success: false })
|
|
329
|
+
return { success: false, text, steps: step }
|
|
330
|
+
}
|
|
331
|
+
if (!this.#onAskUser) {
|
|
332
|
+
resultMessage = 'ask_user is unavailable (no handler configured). Proceed without it or call done.'
|
|
333
|
+
} else {
|
|
334
|
+
try {
|
|
335
|
+
const answer = await this.#onAskUser(input.question)
|
|
336
|
+
resultMessage = `User answered: "${answer}". Use this EXACT value in your next action (e.g. input_text). Do not ask for it again.`
|
|
337
|
+
} catch {
|
|
338
|
+
resultMessage = 'User did not answer.'
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else {
|
|
342
|
+
try {
|
|
343
|
+
const result = await this.#controller.act(actionName, input)
|
|
344
|
+
resultMessage = result.message
|
|
345
|
+
// Progress on the page (a successful click/type/select/nav)
|
|
346
|
+
// clears the ask-without-progress counter, so the next field's
|
|
347
|
+
// question is fine — forms of any size can be filled.
|
|
348
|
+
if (result.success) this.#asksSinceProgress = 0
|
|
349
|
+
} catch (e) {
|
|
350
|
+
resultMessage = `Action failed: ${e.message}`
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
this.#onActivity({ type: 'executed', step, actionName, resultMessage })
|
|
355
|
+
this.#history.push(
|
|
356
|
+
`<step_${step}> Eval: ${evaluation} | Memory: ${memory} | Goal: ${nextGoal} | ${actionName} → ${resultMessage}`
|
|
357
|
+
)
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return {
|
|
361
|
+
success: false,
|
|
362
|
+
text: `Reached max steps (${this.#maxSteps}) without completing the task.`,
|
|
363
|
+
steps: this.#maxSteps,
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|