@dikshanty94/webnav 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +283 -0
- package/bin/webnav +18 -0
- package/dist/agent/loop.d.ts +59 -0
- package/dist/agent/loop.js +267 -0
- package/dist/agent/server.d.ts +60 -0
- package/dist/agent/server.js +254 -0
- package/dist/cli-help.d.ts +3 -0
- package/dist/cli-help.js +111 -0
- package/dist/cli-spec.d.ts +24 -0
- package/dist/cli-spec.js +460 -0
- package/dist/cli.d.ts +208 -0
- package/dist/cli.js +2558 -0
- package/dist/contract.d.ts +20 -0
- package/dist/contract.js +15 -0
- package/dist/creds.d.ts +46 -0
- package/dist/creds.js +126 -0
- package/dist/dashboard/server.d.ts +137 -0
- package/dist/dashboard/server.js +395 -0
- package/dist/dashboard/shell.d.ts +1 -0
- package/dist/dashboard/shell.js +1343 -0
- package/dist/explorer/analyse.d.ts +25 -0
- package/dist/explorer/analyse.js +39 -0
- package/dist/explorer/diff.d.ts +11 -0
- package/dist/explorer/diff.js +22 -0
- package/dist/explorer/draft.d.ts +81 -0
- package/dist/explorer/draft.js +1658 -0
- package/dist/explorer/fingerprint-page.d.ts +11 -0
- package/dist/explorer/fingerprint-page.js +19 -0
- package/dist/explorer/fingerprint.d.ts +14 -0
- package/dist/explorer/fingerprint.js +18 -0
- package/dist/explorer/infer.d.ts +85 -0
- package/dist/explorer/infer.js +370 -0
- package/dist/explorer/pattern-propose.d.ts +27 -0
- package/dist/explorer/pattern-propose.js +81 -0
- package/dist/explorer/patterns.d.ts +49 -0
- package/dist/explorer/patterns.js +290 -0
- package/dist/explorer/shadow.d.ts +3 -0
- package/dist/explorer/shadow.js +95 -0
- package/dist/graph/coverage.d.ts +33 -0
- package/dist/graph/coverage.js +155 -0
- package/dist/graph/edit.d.ts +47 -0
- package/dist/graph/edit.js +211 -0
- package/dist/graph/frontier.d.ts +24 -0
- package/dist/graph/frontier.js +95 -0
- package/dist/graph/seed.d.ts +23 -0
- package/dist/graph/seed.js +29 -0
- package/dist/graph/show.d.ts +8 -0
- package/dist/graph/show.js +6 -0
- package/dist/graph/teach.d.ts +8 -0
- package/dist/graph/teach.js +10 -0
- package/dist/hosted.d.ts +36 -0
- package/dist/hosted.js +69 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +4 -0
- package/dist/mapstore/record.d.ts +128 -0
- package/dist/mapstore/record.js +224 -0
- package/dist/mapstore/schema.sql +47 -0
- package/dist/mapstore/store.d.ts +108 -0
- package/dist/mapstore/store.js +399 -0
- package/dist/mapstore/types.d.ts +87 -0
- package/dist/mapstore/types.js +29 -0
- package/dist/mcp/server.d.ts +46 -0
- package/dist/mcp/server.js +163 -0
- package/dist/paths.d.ts +10 -0
- package/dist/paths.js +33 -0
- package/dist/playwright/adapter.d.ts +62 -0
- package/dist/playwright/adapter.js +145 -0
- package/dist/playwright/ax-adapter.d.ts +33 -0
- package/dist/playwright/ax-adapter.js +76 -0
- package/dist/playwright/capture.d.ts +2 -0
- package/dist/playwright/capture.js +10 -0
- package/dist/playwright/fingerprint.d.ts +40 -0
- package/dist/playwright/fingerprint.js +139 -0
- package/dist/playwright/profile-lock.d.ts +15 -0
- package/dist/playwright/profile-lock.js +51 -0
- package/dist/playwright/sessions.d.ts +72 -0
- package/dist/playwright/sessions.js +229 -0
- package/dist/playwright/snapshot.d.ts +10 -0
- package/dist/playwright/snapshot.js +52 -0
- package/dist/playwright/throttle.d.ts +14 -0
- package/dist/playwright/throttle.js +76 -0
- package/dist/playwright/video.d.ts +13 -0
- package/dist/playwright/video.js +32 -0
- package/dist/protocol.d.ts +49 -0
- package/dist/protocol.js +3 -0
- package/dist/recorder/agent-session.d.ts +59 -0
- package/dist/recorder/agent-session.js +230 -0
- package/dist/recorder/capture-loop.d.ts +19 -0
- package/dist/recorder/capture-loop.js +40 -0
- package/dist/recorder/coverage.d.ts +22 -0
- package/dist/recorder/coverage.js +49 -0
- package/dist/recorder/hover-probe.d.ts +35 -0
- package/dist/recorder/hover-probe.js +100 -0
- package/dist/recorder/ingest.d.ts +39 -0
- package/dist/recorder/ingest.js +138 -0
- package/dist/recorder/live-record.d.ts +41 -0
- package/dist/recorder/live-record.js +255 -0
- package/dist/recorder/live.d.ts +53 -0
- package/dist/recorder/live.js +400 -0
- package/dist/recorder/probe.d.ts +14 -0
- package/dist/recorder/probe.js +33 -0
- package/dist/recorder/replay.d.ts +60 -0
- package/dist/recorder/replay.js +401 -0
- package/dist/recorder/review.d.ts +73 -0
- package/dist/recorder/review.js +171 -0
- package/dist/recorder/snapshot-dom.d.ts +7 -0
- package/dist/recorder/snapshot-dom.js +25 -0
- package/dist/router/auth-status.d.ts +20 -0
- package/dist/router/auth-status.js +55 -0
- package/dist/router/browse.d.ts +108 -0
- package/dist/router/browse.js +214 -0
- package/dist/router/catalog.d.ts +18 -0
- package/dist/router/catalog.js +6 -0
- package/dist/router/extract-content.d.ts +13 -0
- package/dist/router/extract-content.js +87 -0
- package/dist/router/live-extension-browser.d.ts +35 -0
- package/dist/router/live-extension-browser.js +175 -0
- package/dist/router/path.d.ts +7 -0
- package/dist/router/path.js +55 -0
- package/dist/router/read.d.ts +26 -0
- package/dist/router/read.js +30 -0
- package/dist/router/readiness.d.ts +9 -0
- package/dist/router/readiness.js +55 -0
- package/dist/router/replay.d.ts +14 -0
- package/dist/router/replay.js +23 -0
- package/dist/router/resolve.d.ts +21 -0
- package/dist/router/resolve.js +33 -0
- package/dist/router/search-live.d.ts +21 -0
- package/dist/router/search-live.js +98 -0
- package/dist/router/search-providers.d.ts +5 -0
- package/dist/router/search-providers.js +12 -0
- package/dist/router/search.d.ts +6 -0
- package/dist/router/search.js +62 -0
- package/dist/router/suite.d.ts +69 -0
- package/dist/router/suite.js +226 -0
- package/dist/router/tokens.d.ts +13 -0
- package/dist/router/tokens.js +26 -0
- package/dist/router/walk-live.d.ts +57 -0
- package/dist/router/walk-live.js +308 -0
- package/dist/router/walk-session.d.ts +49 -0
- package/dist/router/walk-session.js +90 -0
- package/dist/router/walk.d.ts +48 -0
- package/dist/router/walk.js +384 -0
- package/package.json +59 -0
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { walkRoute } from '../router/walk.js';
|
|
3
|
+
import { findPath } from '../router/path.js';
|
|
4
|
+
import { matchState } from '../explorer/fingerprint.js';
|
|
5
|
+
import { parseSnapshot } from '../playwright/snapshot.js';
|
|
6
|
+
function text(t) {
|
|
7
|
+
return { content: [{ type: 'text', text: t }] };
|
|
8
|
+
}
|
|
9
|
+
// Friendly narrate labels for the panel (crit #3b). The SDK exposes webnav's tools as
|
|
10
|
+
// an in-process MCP server, so tool_use block names arrive as `mcp__webnav__<tool>`;
|
|
11
|
+
// that raw string must never reach the user. Strip the prefix and humanize the couple
|
|
12
|
+
// of internal-sounding verbs (click/type/goto already read fine). The panel also strips
|
|
13
|
+
// defensively — this keeps the label clean at the source.
|
|
14
|
+
function friendlyLabel(raw) {
|
|
15
|
+
const name = raw.replace(/^mcp__webnav__/, '');
|
|
16
|
+
const friendly = {
|
|
17
|
+
check_route: 'checking the map',
|
|
18
|
+
get_page_ax: 'reading the page',
|
|
19
|
+
list_routes: 'listing known routes',
|
|
20
|
+
};
|
|
21
|
+
return friendly[name] ?? name;
|
|
22
|
+
}
|
|
23
|
+
// Build the webnav tools. Each handler drives the injected browser/store; before
|
|
24
|
+
// returning it emits a human-readable `narrate` line so the panel shows what the
|
|
25
|
+
// agent did. This is DISPLAY-ONLY — the browser's own dispatch (the real CDP
|
|
26
|
+
// commands) goes over the channel as `action` events (server.ts), never from here.
|
|
27
|
+
function buildTools(args) {
|
|
28
|
+
const { browser, store, states, emit } = args;
|
|
29
|
+
return [
|
|
30
|
+
{
|
|
31
|
+
name: 'get_page_ax',
|
|
32
|
+
description: 'Return the current page as an accessibility snapshot (YAML). Read this to see refs before clicking or typing.',
|
|
33
|
+
shape: {},
|
|
34
|
+
handler: async () => {
|
|
35
|
+
const snap = await browser.snapshot();
|
|
36
|
+
emit({ type: 'narrate', label: friendlyLabel('get_page_ax'), detail: 'read page' });
|
|
37
|
+
return text(snap);
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: 'click',
|
|
42
|
+
description: 'Click the element with the given ref (from get_page_ax).',
|
|
43
|
+
shape: { ref: z.string().describe('element ref, e.g. e5') },
|
|
44
|
+
handler: async (a) => {
|
|
45
|
+
const ref = String(a.ref);
|
|
46
|
+
await browser.act(ref, null);
|
|
47
|
+
emit({ type: 'narrate', label: 'click', detail: ref });
|
|
48
|
+
return text('clicked ' + ref);
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
name: 'type',
|
|
53
|
+
description: 'Type text into the field with the given ref.',
|
|
54
|
+
shape: { ref: z.string().describe('field ref'), text: z.string().describe('text to type') },
|
|
55
|
+
handler: async (a) => {
|
|
56
|
+
const ref = String(a.ref);
|
|
57
|
+
const val = String(a.text ?? '');
|
|
58
|
+
if (!browser.typeText) {
|
|
59
|
+
return text('cannot type: this browser has no free-text input; field ' + ref + ' was NOT filled');
|
|
60
|
+
}
|
|
61
|
+
await browser.typeText(ref, val);
|
|
62
|
+
emit({ type: 'narrate', label: 'type', detail: '"' + val + '" into ' + ref });
|
|
63
|
+
return text('typed "' + val + '" into ' + ref);
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
name: 'goto',
|
|
68
|
+
description: 'Navigate the tab directly to a URL.',
|
|
69
|
+
shape: { url: z.string().describe('absolute URL') },
|
|
70
|
+
handler: async (a) => {
|
|
71
|
+
const url = String(a.url);
|
|
72
|
+
if (!browser.goto)
|
|
73
|
+
return text('this browser cannot goto a URL');
|
|
74
|
+
// Two modes: Ask gated the whole run up front; Act drives freely. Commit points
|
|
75
|
+
// (Place Order / Pay / Delete) are STILL never auto-fired — walkRoute halts on them
|
|
76
|
+
// regardless of mode — so autonomy here is bounded by that hard safety floor.
|
|
77
|
+
await browser.goto(url, null);
|
|
78
|
+
emit({ type: 'narrate', label: 'goto', detail: url });
|
|
79
|
+
return text('navigated to ' + url);
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
name: 'scroll',
|
|
84
|
+
description: 'Scroll the current page by dy pixels (positive = down, negative = up) to reveal content below or above the fold, then re-read with get_page_ax.',
|
|
85
|
+
shape: { dy: z.number().describe('pixels to scroll; positive scrolls down, negative up (e.g. 600)') },
|
|
86
|
+
handler: async (a) => {
|
|
87
|
+
const dy = Number(a.dy);
|
|
88
|
+
if (!browser.scroll)
|
|
89
|
+
return text('this browser cannot scroll');
|
|
90
|
+
await browser.scroll(dy);
|
|
91
|
+
emit({ type: 'narrate', label: 'scroll', detail: (dy >= 0 ? 'down ' : 'up ') + Math.abs(dy) + 'px' });
|
|
92
|
+
return text('scrolled ' + (dy >= 0 ? 'down ' : 'up ') + Math.abs(dy) + 'px');
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
name: 'list_routes',
|
|
97
|
+
description: 'List the recallable destination states webnav already knows for the CURRENT site. Call this FIRST to discover which goal state ids exist, then pass one to check_route. Returns id + name per destination.',
|
|
98
|
+
shape: {},
|
|
99
|
+
handler: async () => {
|
|
100
|
+
// ZERO-LLM: pure filtering of the in-memory states. Current site = the nodeId of
|
|
101
|
+
// whatever known state the live page matches (fingerprint-based, so account-id URL
|
|
102
|
+
// differences never matter). If the page matches no state, fall back to ALL sites'
|
|
103
|
+
// destinations so the agent still sees what maps exist. A real destination has a
|
|
104
|
+
// NON-EMPTY fingerprint (mirrors matchState: _shell + empty-fp stubs identify nothing).
|
|
105
|
+
const match = matchState(parseSnapshot(await browser.snapshot()), states);
|
|
106
|
+
const site = match.status === 'matched' ? match.state.nodeId : null;
|
|
107
|
+
const dests = states.filter((s) => s.fingerprint.length > 0 && (site === null || s.nodeId === site));
|
|
108
|
+
emit({ type: 'narrate', label: friendlyLabel('list_routes'), detail: dests.length + ' destination(s)' });
|
|
109
|
+
if (dests.length === 0)
|
|
110
|
+
return text('no recallable destinations known for this site — drive manually with click/type/goto.');
|
|
111
|
+
const lines = dests.map((s) => '- ' + s.id + ' — ' + s.semanticName);
|
|
112
|
+
return text('known destinations (pass an id to check_route):\n' + lines.join('\n'));
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
name: 'check_route',
|
|
117
|
+
description: 'RECALL-FIRST: check whether the map already knows a deterministic route from the current page to a goal state. On a hit, webnav walks it for you and reports the result; on a miss, drive manually with click/type/goto.',
|
|
118
|
+
shape: { goalStateId: z.string().describe('id of the destination state, e.g. sd:cart') },
|
|
119
|
+
handler: async (a) => {
|
|
120
|
+
const goalStateId = String(a.goalStateId);
|
|
121
|
+
// ZERO-LLM: parse the live snapshot, match it to a known state, ask findPath.
|
|
122
|
+
const nodes = parseSnapshot(await browser.snapshot());
|
|
123
|
+
const match = matchState(nodes, states);
|
|
124
|
+
if (match.status !== 'matched') {
|
|
125
|
+
emit({ type: 'narrate', label: friendlyLabel('check_route'), detail: 'current page not on a known state' });
|
|
126
|
+
return text('no route: current page is not on a known map state — drive manually with click/type/goto.');
|
|
127
|
+
}
|
|
128
|
+
const start = match.state.id;
|
|
129
|
+
const path = findPath(store, start, goalStateId);
|
|
130
|
+
if (!path) {
|
|
131
|
+
emit({ type: 'narrate', label: friendlyLabel('check_route'), detail: 'no route ' + start + ' -> ' + goalStateId });
|
|
132
|
+
return text('no route from ' + start + ' to ' + goalStateId + ' in the map — drive manually.');
|
|
133
|
+
}
|
|
134
|
+
emit({ type: 'narrate', label: friendlyLabel('check_route'), detail: path.join(' -> ') });
|
|
135
|
+
// THE PAYOFF: hand the found route to the deterministic replay (walkRoute
|
|
136
|
+
// UNMODIFIED), then narrate its terminal RecallResponse back to the agent.
|
|
137
|
+
const res = await walkRoute({
|
|
138
|
+
goalName: 'agent:' + args.goal,
|
|
139
|
+
startStateId: start,
|
|
140
|
+
goalStateId,
|
|
141
|
+
store,
|
|
142
|
+
states,
|
|
143
|
+
browser,
|
|
144
|
+
path,
|
|
145
|
+
});
|
|
146
|
+
return text('route found [' + path.join(' -> ') + ']; walked: ' + summarizeRecall(res));
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
];
|
|
150
|
+
}
|
|
151
|
+
// One-line summary of a walk's terminal response for the agent to read as a tool result.
|
|
152
|
+
function summarizeRecall(res) {
|
|
153
|
+
switch (res.status) {
|
|
154
|
+
case 'done': return 'done';
|
|
155
|
+
case 'needs-navigation': return 'needs-navigation at step ' + res.at + ': ' + res.question;
|
|
156
|
+
case 'needs-classification': return 'needs-classification: ' + res.action;
|
|
157
|
+
case 'needs-auth': return 'needs-auth (login required) at ' + res.loginUrl;
|
|
158
|
+
case 'checkpoint': return 'checkpoint at ' + res.state;
|
|
159
|
+
case 'failed': return 'failed: ' + res.reason;
|
|
160
|
+
default: return res.status;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// The default QueryFn: wire the tools into a real SDK MCP server and run query().
|
|
164
|
+
// Imported lazily so unit tests (which inject a fake) never load the SDK.
|
|
165
|
+
const defaultQuery = async function* ({ prompt, tools, model, resume, signal }) {
|
|
166
|
+
const { tool, createSdkMcpServer, query } = await import('@anthropic-ai/claude-agent-sdk');
|
|
167
|
+
const sdkTools = tools.map((t) => tool(t.name, t.description, t.shape, async (a, extra) => t.handler(a, extra)));
|
|
168
|
+
const server = createSdkMcpServer({ name: 'webnav', tools: sdkTools });
|
|
169
|
+
const allowedTools = tools.map((t) => 'mcp__webnav__' + t.name);
|
|
170
|
+
yield* query({
|
|
171
|
+
prompt,
|
|
172
|
+
options: {
|
|
173
|
+
mcpServers: { webnav: server },
|
|
174
|
+
allowedTools,
|
|
175
|
+
permissionMode: 'default',
|
|
176
|
+
// Only pass `model` when the user picked one; omit → SDK default (never '').
|
|
177
|
+
...(model ? { model } : {}),
|
|
178
|
+
// Continue a prior conversation (Options.resume) when a session id is carried over.
|
|
179
|
+
...(resume ? { resume } : {}),
|
|
180
|
+
...(signal ? { abortController: abortFromSignal(signal) } : {}),
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
};
|
|
184
|
+
// The SDK wants an AbortController; the loop is handed an AbortSignal (for /stop).
|
|
185
|
+
// Bridge them so aborting the signal aborts the query.
|
|
186
|
+
function abortFromSignal(signal) {
|
|
187
|
+
const ctrl = new AbortController();
|
|
188
|
+
if (signal.aborted)
|
|
189
|
+
ctrl.abort();
|
|
190
|
+
else
|
|
191
|
+
signal.addEventListener('abort', () => ctrl.abort(), { once: true });
|
|
192
|
+
return ctrl;
|
|
193
|
+
}
|
|
194
|
+
const SYSTEM = [
|
|
195
|
+
'You navigate a live browser tab to accomplish the user goal.',
|
|
196
|
+
'RECALL FIRST: call list_routes to discover the destination state ids webnav already knows for this site, then call check_route with the relevant id — if the map knows a route, webnav walks it deterministically and reports the result. You do NOT need to guess ids; list_routes shows the real ones.',
|
|
197
|
+
'Only if list_routes is empty or check_route reports no route: call get_page_ax to see the page, then drive manually with click/type/goto by ref.',
|
|
198
|
+
'Irreversible actions (Place Order, Pay, Delete) are never fired automatically — if the walk hits one it stops and hands back to you.',
|
|
199
|
+
].join(' ');
|
|
200
|
+
/**
|
|
201
|
+
* Run one agent goal. Streams SDK narration back as AgentEvents:
|
|
202
|
+
* - assistant text -> { type: 'turn', text }
|
|
203
|
+
* - tool-use blocks -> { type: 'narrate', label } (display-only; the REAL CDP `action`
|
|
204
|
+
* events come from the channel in server.ts, never from here)
|
|
205
|
+
* - final result -> { type: 'done', summary }
|
|
206
|
+
* - any throw -> { type: 'error', message }
|
|
207
|
+
*
|
|
208
|
+
* Ask/Auto/Act gate semantics (crit #3/#4 — these three are OBSERVABLY different):
|
|
209
|
+
* - ACT: no confirmations. Drives freely. (Commit points are still protected by
|
|
210
|
+
* walkRoute's needs-classification — never auto-fired.)
|
|
211
|
+
* - AUTO: drives freely EXCEPT it pauses for approval before a goto to a NEW ORIGIN
|
|
212
|
+
* (cross-site navigation = the cheap "meaningful action" signal). Gate lives in
|
|
213
|
+
* the goto tool handler (buildTools). Same-origin driving is never gated.
|
|
214
|
+
* - ASK: emit the plan, then BLOCK the whole run on awaitApproval() BEFORE the query's
|
|
215
|
+
* first driving tool can run. Approve → run. Deny → emit a "denied" done and
|
|
216
|
+
* return without ever starting the query (nothing drives).
|
|
217
|
+
*/
|
|
218
|
+
export async function runAgentGoal(args) {
|
|
219
|
+
const { goal, mode, emit } = args;
|
|
220
|
+
const query = args.query ?? defaultQuery;
|
|
221
|
+
const tools = buildTools(args);
|
|
222
|
+
// Turn 1 (fresh): send SYSTEM + goal. Resumed turn: send JUST the goal — the SDK
|
|
223
|
+
// already holds the system prompt + history from the resumed session, so re-injecting
|
|
224
|
+
// SYSTEM would make Claude re-orient and re-list routes every message (the whole bug).
|
|
225
|
+
const prompt = args.resumeSessionId ? goal : SYSTEM + '\n\nGoal: ' + goal;
|
|
226
|
+
if (mode === 'ask') {
|
|
227
|
+
emit({ type: 'plan', steps: ['Check the map for a known route, then drive the tab step by step toward: ' + goal] });
|
|
228
|
+
// HARD GATE: block before the first drive. The SDK query does not start until the
|
|
229
|
+
// user approves — that is what makes Ask genuinely different from Auto/Act.
|
|
230
|
+
const ok = args.awaitApproval ? await args.awaitApproval() : true;
|
|
231
|
+
if (!ok) {
|
|
232
|
+
emit({ type: 'done', summary: 'denied — nothing was done' });
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
try {
|
|
237
|
+
let finalText = '';
|
|
238
|
+
let sdkSessionSeen = false;
|
|
239
|
+
for await (const msg of query({ prompt, tools, mode, model: args.model, resume: args.resumeSessionId, emit, signal: args.signal })) {
|
|
240
|
+
const m = msg;
|
|
241
|
+
// Capture the SDK session id off the message stream (every SDKMessage carries it).
|
|
242
|
+
// Report it ONCE so the caller can persist it and resume this conversation next turn.
|
|
243
|
+
if (!sdkSessionSeen && typeof m.session_id === 'string' && m.session_id) {
|
|
244
|
+
sdkSessionSeen = true;
|
|
245
|
+
args.onSdkSession?.(m.session_id);
|
|
246
|
+
}
|
|
247
|
+
if (m.type === 'assistant') {
|
|
248
|
+
for (const block of m.message?.content ?? []) {
|
|
249
|
+
if (block.type === 'text' && block.text)
|
|
250
|
+
emit({ type: 'turn', text: block.text });
|
|
251
|
+
// NOTE: we deliberately do NOT narrate the tool_use block here. Each tool's
|
|
252
|
+
// handler (buildTools) emits its OWN narrate with the RESULT detail (e.g.
|
|
253
|
+
// "listing known routes: 46 destination(s)"); narrating the announcement too
|
|
254
|
+
// produced a duplicate bare line per action. The handler narrate is the one.
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
else if (m.type === 'result') {
|
|
258
|
+
if (m.subtype === 'success')
|
|
259
|
+
finalText = m.result ?? finalText;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
emit({ type: 'done', summary: finalText || undefined });
|
|
263
|
+
}
|
|
264
|
+
catch (e) {
|
|
265
|
+
emit({ type: 'error', message: e instanceof Error ? e.message : String(e) });
|
|
266
|
+
}
|
|
267
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import type { AgentChannel } from '../router/live-extension-browser.js';
|
|
3
|
+
import type { RecordStore } from '../mapstore/record.js';
|
|
4
|
+
export interface ScreencastFrame {
|
|
5
|
+
data: string;
|
|
6
|
+
timestampMs: number;
|
|
7
|
+
}
|
|
8
|
+
export type AgentCommand = {
|
|
9
|
+
kind: 'get-ax';
|
|
10
|
+
} | {
|
|
11
|
+
kind: 'click';
|
|
12
|
+
nodeId: string;
|
|
13
|
+
} | {
|
|
14
|
+
kind: 'type';
|
|
15
|
+
nodeId: string;
|
|
16
|
+
text?: string;
|
|
17
|
+
} | {
|
|
18
|
+
kind: 'goto';
|
|
19
|
+
url: string;
|
|
20
|
+
} | {
|
|
21
|
+
kind: 'scroll';
|
|
22
|
+
dy: number;
|
|
23
|
+
} | {
|
|
24
|
+
kind: 'current-url';
|
|
25
|
+
};
|
|
26
|
+
export type AgentEvent = {
|
|
27
|
+
type: 'turn';
|
|
28
|
+
text: string;
|
|
29
|
+
} | {
|
|
30
|
+
type: 'narrate';
|
|
31
|
+
label: string;
|
|
32
|
+
detail?: string;
|
|
33
|
+
} | {
|
|
34
|
+
type: 'action';
|
|
35
|
+
id: string;
|
|
36
|
+
cmd: AgentCommand;
|
|
37
|
+
} | {
|
|
38
|
+
type: 'done';
|
|
39
|
+
summary?: string;
|
|
40
|
+
} | {
|
|
41
|
+
type: 'error';
|
|
42
|
+
message: string;
|
|
43
|
+
} | {
|
|
44
|
+
type: 'plan';
|
|
45
|
+
steps: string[];
|
|
46
|
+
};
|
|
47
|
+
export interface AgentGoalBody {
|
|
48
|
+
goal: string;
|
|
49
|
+
sessionId: string;
|
|
50
|
+
mode: string;
|
|
51
|
+
model?: string;
|
|
52
|
+
newChat?: boolean;
|
|
53
|
+
}
|
|
54
|
+
export interface ServeAgentOpts {
|
|
55
|
+
onGoal?: (goal: AgentGoalBody, channel: AgentChannel, emit: (e: AgentEvent) => void, awaitApproval: () => Promise<boolean>, signal: AbortSignal, resumeSessionId: string | undefined, onSdkSession: (sdkSessionId: string) => void) => Promise<void>;
|
|
56
|
+
onScreencast?: (sessionId: string, frames: ScreencastFrame[]) => Promise<void>;
|
|
57
|
+
commandTimeoutMs?: number;
|
|
58
|
+
token?: string;
|
|
59
|
+
}
|
|
60
|
+
export declare function serveAgent(port: number, store: RecordStore, opts?: ServeAgentOpts): http.Server;
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import crypto from 'node:crypto';
|
|
3
|
+
import { ingestAX } from '../recorder/ingest.js';
|
|
4
|
+
// Minimal drivable landing page (served unauthenticated at GET /landing). On-brand
|
|
5
|
+
// (the webnav map-pin), both light + dark via prefers-color-scheme, self-contained (the
|
|
6
|
+
// extension CSP + a bare browser tab: no external anything). It's just a neutral,
|
|
7
|
+
// debugger-attachable place for the agent to start when the launch tab wasn't drivable —
|
|
8
|
+
// the agent navigates onward from here immediately.
|
|
9
|
+
const LANDING_HTML = `<!doctype html><html lang="en"><head><meta charset="utf-8">
|
|
10
|
+
<meta name="viewport" content="width=device-width,initial-scale=1"><title>webnav — ready</title>
|
|
11
|
+
<style>
|
|
12
|
+
:root { color-scheme: light dark; --bg:#eef1f6; --ink:#1a2230; --ink-soft:#4a5568; --accent:#2f6fe0; }
|
|
13
|
+
@media (prefers-color-scheme: dark) { :root { --bg:#0d1017; --ink:#e7ecf4; --ink-soft:#a4aec0; --accent:#5b9dff; } }
|
|
14
|
+
html,body { height:100%; margin:0; }
|
|
15
|
+
body { display:flex; flex-direction:column; align-items:center; justify-content:center; gap:14px;
|
|
16
|
+
background:var(--bg); color:var(--ink); font:15px/1.5 ui-sans-serif,-apple-system,system-ui,sans-serif; }
|
|
17
|
+
svg { width:40px; height:40px; }
|
|
18
|
+
h1 { font-size:16px; font-weight:650; margin:0; letter-spacing:-.01em; }
|
|
19
|
+
p { margin:0; color:var(--ink-soft); font-size:13px; }
|
|
20
|
+
</style></head><body>
|
|
21
|
+
<svg viewBox="0 0 24 24" fill="none" stroke="var(--accent)" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round">
|
|
22
|
+
<path d="M12 22c4.5-5 7-8.4 7-12A7 7 0 0 0 5 10c0 3.6 2.5 7 7 12z"/><circle cx="12" cy="10" r="2.6" fill="var(--accent)" stroke="none"/>
|
|
23
|
+
</svg>
|
|
24
|
+
<h1>webnav — ready</h1>
|
|
25
|
+
<p>Give the agent a goal in the side panel; it will navigate from here.</p>
|
|
26
|
+
</body></html>`;
|
|
27
|
+
export function serveAgent(port, store, opts = {}) {
|
|
28
|
+
const commandTimeoutMs = opts.commandTimeoutMs ?? 30_000;
|
|
29
|
+
// `||` not `??`: an explicitly-passed empty string must still fall back to a random
|
|
30
|
+
// token — token:'' would make authorized('') true and disable auth entirely.
|
|
31
|
+
const token = opts.token || crypto.randomBytes(16).toString('hex');
|
|
32
|
+
// LAST-CONNECTION-WINS (single-panel guard): two open panels (or a reopened panel
|
|
33
|
+
// racing the old EventSource's close) must not both receive+execute every `action` —
|
|
34
|
+
// that double-fires CDP clicks. Only the newest /events connection is kept live; a
|
|
35
|
+
// fresh connection evicts+ends whatever was there before.
|
|
36
|
+
let current = null;
|
|
37
|
+
const emit = (e) => current?.send(e);
|
|
38
|
+
const pending = new Map();
|
|
39
|
+
let seq = 0;
|
|
40
|
+
// Single outstanding approval gate (one goal runs at a time). onGoal is handed an
|
|
41
|
+
// awaitApproval() that parks here until /approve (or /stop, which denies) resolves it.
|
|
42
|
+
// ponytail: one slot, not a per-goal map — the server drives one goal at a time.
|
|
43
|
+
let pendingApproval = null;
|
|
44
|
+
// Single outstanding run's abort controller (one goal at a time, same as the approval
|
|
45
|
+
// slot). /stop aborts it so the SDK query is cancelled, not just the pending commands.
|
|
46
|
+
let currentGoalAbort = null;
|
|
47
|
+
// Panel conversation key → last SDK session id, so a follow-up goal resumes the same
|
|
48
|
+
// conversation (Claude remembers what it did + the page it's on). Keyed by the panel's
|
|
49
|
+
// `sessionId`; the New-chat button rotates that key (or sends newChat:true) → fresh.
|
|
50
|
+
// ponytail: in-memory Map — one process per agent-serve, no persistence needed.
|
|
51
|
+
const sdkSessions = new Map();
|
|
52
|
+
function awaitApproval() {
|
|
53
|
+
// A fresh goal supersedes any stale unresolved gate (deny it so nothing leaks).
|
|
54
|
+
if (pendingApproval)
|
|
55
|
+
pendingApproval(false);
|
|
56
|
+
return new Promise((resolve) => { pendingApproval = resolve; });
|
|
57
|
+
}
|
|
58
|
+
function resolveApproval(approved) {
|
|
59
|
+
const p = pendingApproval;
|
|
60
|
+
pendingApproval = null;
|
|
61
|
+
p?.(approved);
|
|
62
|
+
}
|
|
63
|
+
function rejectAllPending(message) {
|
|
64
|
+
for (const [id, p] of pending) {
|
|
65
|
+
clearTimeout(p.timer);
|
|
66
|
+
p.reject(new Error(message));
|
|
67
|
+
pending.delete(id);
|
|
68
|
+
}
|
|
69
|
+
resolveApproval(false); // /stop also denies an outstanding approval gate
|
|
70
|
+
}
|
|
71
|
+
// The real AgentChannel: every call becomes an `action` SSE event, resolved when
|
|
72
|
+
// the extension POSTs to /api/agent/command-result (or timed out).
|
|
73
|
+
function makeChannel() {
|
|
74
|
+
function dispatchCommand(cmd) {
|
|
75
|
+
const id = 'cmd' + (++seq);
|
|
76
|
+
return new Promise((resolve, reject) => {
|
|
77
|
+
const timer = setTimeout(() => {
|
|
78
|
+
pending.delete(id);
|
|
79
|
+
reject(new Error(`agent-serve: command "${id}" timed out after ${commandTimeoutMs}ms`));
|
|
80
|
+
}, commandTimeoutMs);
|
|
81
|
+
pending.set(id, { resolve, reject, timer });
|
|
82
|
+
emit({ type: 'action', id, cmd });
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
getAX: () => dispatchCommand({ kind: 'get-ax' }),
|
|
87
|
+
dispatch: async (cmd) => { await dispatchCommand(cmd); },
|
|
88
|
+
goto: async (url) => { await dispatchCommand({ kind: 'goto', url }); },
|
|
89
|
+
scroll: async (dy) => { await dispatchCommand({ kind: 'scroll', dy }); },
|
|
90
|
+
currentUrl: () => dispatchCommand({ kind: 'current-url' }),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
// Constant-time token check. The extension sends the secret as an `x-webnav-token`
|
|
94
|
+
// header on POSTs; EventSource can't set headers, so the SSE GET carries it as a
|
|
95
|
+
// `?token=` query param. No CORS headers are emitted at all — this server is only
|
|
96
|
+
// reached by the extension (a privileged context whose fetch/EventSource are NOT
|
|
97
|
+
// page-CORS-gated), so `*` was never needed; dropping it blocks any web page from
|
|
98
|
+
// reading responses cross-origin, and the token blocks driving the browser without
|
|
99
|
+
// the secret (closes the DNS-rebind / local-process attack).
|
|
100
|
+
const authorized = (req) => {
|
|
101
|
+
const url = new URL(req.url ?? '', 'http://x');
|
|
102
|
+
const presented = req.headers['x-webnav-token'] ?? url.searchParams.get('token') ?? '';
|
|
103
|
+
const a = Buffer.from(presented);
|
|
104
|
+
const b = Buffer.from(token);
|
|
105
|
+
return a.length === b.length && crypto.timingSafeEqual(a, b);
|
|
106
|
+
};
|
|
107
|
+
const server = http.createServer((req, res) => {
|
|
108
|
+
const sendJson = (code, body) => res.writeHead(code, { 'content-type': 'application/json' }).end(JSON.stringify(body));
|
|
109
|
+
// Drivable landing page — the one route that is NOT token-gated. When the agent must
|
|
110
|
+
// open a fresh tab (the user launched from a chrome:// / New Tab page that CDP can't
|
|
111
|
+
// drive), it lands HERE instead of an arbitrary google.com: a minimal, on-brand,
|
|
112
|
+
// http(s) (therefore debugger-attachable) page. It's a static page with NO capability,
|
|
113
|
+
// NO data, NO command surface — safe to serve unauthenticated (a browser tab can't send
|
|
114
|
+
// the x-webnav-token header anyway). The agent immediately goto()s to the real target.
|
|
115
|
+
if (req.method === 'GET' && (req.url === '/landing' || req.url === '/')) {
|
|
116
|
+
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
|
|
117
|
+
res.end(LANDING_HTML);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (!authorized(req))
|
|
121
|
+
return sendJson(401, { ok: false, error: 'unauthorized' });
|
|
122
|
+
if (req.method === 'GET' && (req.url ?? '').startsWith('/api/agent/events')) {
|
|
123
|
+
// Evict whatever connection was previously current — last-connection-wins.
|
|
124
|
+
current?.res.end();
|
|
125
|
+
res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' });
|
|
126
|
+
res.write(': connected\n\n');
|
|
127
|
+
const send = (e) => res.write('data: ' + JSON.stringify(e) + '\n\n');
|
|
128
|
+
const self = { send, res };
|
|
129
|
+
current = self;
|
|
130
|
+
const beat = setInterval(() => res.write(': ping\n\n'), 15000);
|
|
131
|
+
req.on('close', () => { clearInterval(beat); if (current === self)
|
|
132
|
+
current = null; });
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
if (req.method === 'POST' && req.url === '/api/agent/goal') {
|
|
136
|
+
readBody(req).then((raw) => {
|
|
137
|
+
let body;
|
|
138
|
+
try {
|
|
139
|
+
body = JSON.parse(raw);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return sendJson(400, { ok: false, error: 'invalid JSON body' });
|
|
143
|
+
}
|
|
144
|
+
if (!body.goal || !body.sessionId)
|
|
145
|
+
return sendJson(400, { ok: false, error: 'goal and sessionId required' });
|
|
146
|
+
sendJson(200, { ok: true });
|
|
147
|
+
if (opts.onGoal) {
|
|
148
|
+
const channel = makeChannel();
|
|
149
|
+
// A fresh goal supersedes any prior run's abort controller.
|
|
150
|
+
const abort = new AbortController();
|
|
151
|
+
currentGoalAbort = abort;
|
|
152
|
+
// New-chat: forget the prior SDK conversation for this key → this run cold-starts.
|
|
153
|
+
if (body.newChat)
|
|
154
|
+
sdkSessions.delete(body.sessionId);
|
|
155
|
+
const resumeSessionId = sdkSessions.get(body.sessionId);
|
|
156
|
+
const onSdkSession = (id) => { sdkSessions.set(body.sessionId, id); };
|
|
157
|
+
opts.onGoal(body, channel, emit, awaitApproval, abort.signal, resumeSessionId, onSdkSession)
|
|
158
|
+
.catch((e) => emit({ type: 'error', message: String(e) }))
|
|
159
|
+
.finally(() => { if (currentGoalAbort === abort)
|
|
160
|
+
currentGoalAbort = null; });
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
if (req.method === 'POST' && req.url === '/api/agent/command-result') {
|
|
166
|
+
readBody(req).then((raw) => {
|
|
167
|
+
let body;
|
|
168
|
+
try {
|
|
169
|
+
body = JSON.parse(raw);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return sendJson(400, { ok: false, error: 'invalid JSON body' });
|
|
173
|
+
}
|
|
174
|
+
const p = body.id ? pending.get(body.id) : undefined;
|
|
175
|
+
if (!p)
|
|
176
|
+
return sendJson(404, { ok: false, error: 'unknown command id' });
|
|
177
|
+
clearTimeout(p.timer);
|
|
178
|
+
pending.delete(body.id);
|
|
179
|
+
p.resolve(body.result);
|
|
180
|
+
sendJson(200, { ok: true });
|
|
181
|
+
});
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (req.method === 'POST' && req.url === '/api/agent/approve') {
|
|
185
|
+
readBody(req).then((raw) => {
|
|
186
|
+
let body;
|
|
187
|
+
try {
|
|
188
|
+
body = JSON.parse(raw);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return sendJson(400, { ok: false, error: 'invalid JSON body' });
|
|
192
|
+
}
|
|
193
|
+
resolveApproval(body.approved === true);
|
|
194
|
+
sendJson(200, { ok: true });
|
|
195
|
+
});
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (req.method === 'POST' && req.url === '/api/agent/stop') {
|
|
199
|
+
// Stop must ACTUALLY stop: reject in-flight browser commands AND abort the SDK
|
|
200
|
+
// query (via the per-goal signal threaded to runAgentGoal). Rejecting pending
|
|
201
|
+
// commands alone only fails the next browser call — the SDK turn keeps running.
|
|
202
|
+
rejectAllPending('agent-serve: stopped');
|
|
203
|
+
currentGoalAbort?.abort();
|
|
204
|
+
sendJson(200, { ok: true });
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
if (req.method === 'POST' && req.url === '/ingest-ax') {
|
|
208
|
+
readBody(req).then((raw) => {
|
|
209
|
+
try {
|
|
210
|
+
const body = JSON.parse(raw);
|
|
211
|
+
if (!body.sessionId || !Array.isArray(body.steps))
|
|
212
|
+
throw new Error('sessionId and steps[] required');
|
|
213
|
+
const appended = ingestAX(body, store);
|
|
214
|
+
sendJson(200, { ok: true, appended });
|
|
215
|
+
}
|
|
216
|
+
catch (e) {
|
|
217
|
+
sendJson(400, { ok: false, error: String(e) });
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (req.method === 'POST' && req.url === '/api/agent/screencast') {
|
|
223
|
+
readBody(req).then(async (raw) => {
|
|
224
|
+
try {
|
|
225
|
+
const body = JSON.parse(raw);
|
|
226
|
+
if (!body.sessionId || !Array.isArray(body.frames))
|
|
227
|
+
throw new Error('sessionId and frames[] required');
|
|
228
|
+
// Empty frames (nothing captured) is a no-op success, not an error.
|
|
229
|
+
if (body.frames.length && opts.onScreencast)
|
|
230
|
+
await opts.onScreencast(body.sessionId, body.frames);
|
|
231
|
+
sendJson(200, { ok: true, frames: body.frames.length });
|
|
232
|
+
}
|
|
233
|
+
catch (e) {
|
|
234
|
+
sendJson(400, { ok: false, error: String(e) });
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
sendJson(404, { ok: false, error: 'not found' });
|
|
240
|
+
});
|
|
241
|
+
server.listen(port, '127.0.0.1');
|
|
242
|
+
return server;
|
|
243
|
+
}
|
|
244
|
+
function readBody(req) {
|
|
245
|
+
return new Promise((resolve, reject) => {
|
|
246
|
+
let data = '';
|
|
247
|
+
// ponytail: 50MB cap mirrors serveIngest's precedent — a goal run can carry
|
|
248
|
+
// full AX trees per step.
|
|
249
|
+
req.on('data', (c) => { data += c; if (data.length > 50_000_000)
|
|
250
|
+
req.destroy(); });
|
|
251
|
+
req.on('end', () => resolve(data));
|
|
252
|
+
req.on('error', reject);
|
|
253
|
+
});
|
|
254
|
+
}
|