@cairnvibe/sdk 0.2.13 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-loop.d.ts +113 -0
- package/dist/agent-loop.js +128 -0
- package/dist/cairn-widget.js +14 -9
- package/dist/cursor-overlay.d.ts +19 -0
- package/dist/cursor-overlay.js +126 -0
- package/dist/element-ladder.d.ts +71 -0
- package/dist/element-ladder.js +168 -0
- package/dist/index.d.ts +79 -1
- package/dist/index.js +886 -96
- package/dist/key-rotator.d.ts +28 -0
- package/dist/key-rotator.js +57 -3
- package/dist/memory-sqlite.d.ts +86 -0
- package/dist/memory-sqlite.js +230 -0
- package/dist/realtime-cli.js +22 -1
- package/dist/realtime-server.d.ts +83 -2
- package/dist/realtime-server.js +561 -121
- package/dist/server.d.ts +266 -5
- package/dist/server.js +1013 -83
- package/dist/skill-store.d.ts +17 -0
- package/dist/skill-store.js +78 -0
- package/dist/tts-stream.d.ts +25 -0
- package/dist/tts-stream.js +32 -0
- package/dist/vad.d.ts +27 -0
- package/dist/vad.js +128 -0
- package/dist/verb-executor.d.ts +32 -11
- package/dist/verb-executor.js +315 -39
- package/dist/webmcp-client.d.ts +14 -1
- package/dist/webmcp-client.js +22 -1
- package/package.json +3 -1
- package/src/agent-loop.ts +222 -0
- package/src/cursor-overlay.ts +130 -0
- package/src/element-ladder.ts +170 -0
- package/src/index.tsx +935 -100
- package/src/key-rotator.ts +57 -2
- package/src/memory-sqlite.ts +283 -0
- package/src/realtime-cli.ts +24 -1
- package/src/realtime-server.ts +669 -123
- package/src/server.ts +1119 -83
- package/src/skill-store.ts +88 -0
- package/src/tts-stream.ts +30 -0
- package/src/vad.ts +153 -0
- package/src/verb-executor.ts +329 -42
- package/src/web-component.ts +97 -24
- package/src/webmcp-client.ts +30 -2
package/dist/verb-executor.js
CHANGED
|
@@ -10,20 +10,35 @@ exports.executeToolStep = executeToolStep;
|
|
|
10
10
|
exports.executeVerbResponse = executeVerbResponse;
|
|
11
11
|
const core_1 = require("@cairnvibe/core");
|
|
12
12
|
const element_ladder_1 = require("./element-ladder");
|
|
13
|
+
const cursor_overlay_1 = require("./cursor-overlay");
|
|
13
14
|
const webmcp_client_1 = require("./webmcp-client");
|
|
15
|
+
// wait_for's own real, bounded retry budget — longer than
|
|
16
|
+
// findElementWithRetry's own default (2 attempts, 300ms apart, ~300ms
|
|
17
|
+
// total), since this verb exists specifically for "I know something
|
|
18
|
+
// async should show up" — a toast, a panel appearing after a click — not
|
|
19
|
+
// the incidental transient-miss recovery findElementWithRetry's default
|
|
20
|
+
// already covers for click/fill/batch steps.
|
|
21
|
+
const WAIT_FOR_ATTEMPTS = 6;
|
|
22
|
+
const WAIT_FOR_DELAY_MS = 500;
|
|
14
23
|
/**
|
|
15
24
|
* Promise wrapper around executeVerbResponse for a continuing verb
|
|
16
|
-
* (click/fill/read/call_tool
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
25
|
+
* (click/fill/read/call_tool, or now a navigate marked `continueAfter` —
|
|
26
|
+
* see isTerminalVerb in @cairnvibe/core) — resolves once the real action
|
|
27
|
+
* has actually finished (synchronously for click/fill/read, after a real
|
|
28
|
+
* await for call_tool/navigate) with its real observation, instead of the
|
|
29
|
+
* fire-and-forget callback shape every other verb uses. This is what a
|
|
30
|
+
* loop driver awaits before deciding whether to call the model again.
|
|
31
|
+
* `onNavigate` is only needed for that new navigate-as-continuing-step
|
|
32
|
+
* case — every existing caller that never passes it keeps working
|
|
33
|
+
* unchanged (a continueAfter navigate with no onNavigate here would just
|
|
34
|
+
* never actually move the page; real callers always pass one, same as
|
|
35
|
+
* handleVerb's own options already do for the terminal case).
|
|
21
36
|
*/
|
|
22
|
-
function executeToolStep(raw, route, liveElements) {
|
|
37
|
+
function executeToolStep(raw, route, liveElements, onNavigate, onConfirmTool) {
|
|
23
38
|
return new Promise((resolve) => {
|
|
24
39
|
// executeVerbResponse only ever reaches onToolStep for a genuinely
|
|
25
40
|
// continuing verb — callers are only expected to call this after
|
|
26
|
-
// already confirming (via
|
|
41
|
+
// already confirming (via isTerminalVerb) that the parsed verb is one,
|
|
27
42
|
// so this should always fire; a real timeout (not an immediate
|
|
28
43
|
// microtask — call_tool's own real network round trip needs the time)
|
|
29
44
|
// is the safety net for the case where it somehow doesn't, so a loop
|
|
@@ -32,6 +47,8 @@ function executeToolStep(raw, route, liveElements) {
|
|
|
32
47
|
executeVerbResponse(raw, route, {
|
|
33
48
|
onExplain: () => { },
|
|
34
49
|
liveElements,
|
|
50
|
+
onNavigate,
|
|
51
|
+
onConfirmTool,
|
|
35
52
|
onToolStep: (result) => {
|
|
36
53
|
clearTimeout(timer);
|
|
37
54
|
resolve(result);
|
|
@@ -62,19 +79,49 @@ function dispatchVerb(verb, route, options) {
|
|
|
62
79
|
return;
|
|
63
80
|
}
|
|
64
81
|
(0, element_ladder_1.highlightElement)(el);
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
82
|
+
// The cursor visibly arrives before "open"'s real click fires — the
|
|
83
|
+
// whole point is a user watching sees where it's about to click
|
|
84
|
+
// BEFORE the menu/modal/panel actually opens, not simultaneously.
|
|
85
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
86
|
+
// "open" means make the thing actually appear (a menu, a modal, a
|
|
87
|
+
// panel) — highlighting alone doesn't do that; a real click does.
|
|
88
|
+
if (verb.verb === "open")
|
|
89
|
+
el.click();
|
|
90
|
+
if (verb.text)
|
|
91
|
+
options.onExplain(verb.text);
|
|
92
|
+
});
|
|
71
93
|
return;
|
|
72
94
|
}
|
|
73
|
-
case "navigate":
|
|
95
|
+
case "navigate": {
|
|
96
|
+
// Real, live-reported gap this closes: navigate used to ALWAYS end
|
|
97
|
+
// the turn the instant it fired, even for a compound goal like "buy
|
|
98
|
+
// earbuds" that needs navigate, then search, then a real report
|
|
99
|
+
// back — see isTerminalVerb's own doc comment in @cairnvibe/core.
|
|
100
|
+
// `options.onToolStep` is only ever set by executeToolStep's own
|
|
101
|
+
// continuing-step wrapper — handleVerb's options never provide it —
|
|
102
|
+
// so this branch can only run when the caller already confirmed
|
|
103
|
+
// (via isTerminalVerb) that this navigate was genuinely marked
|
|
104
|
+
// continueAfter; the defensive `verb.continueAfter` check here is
|
|
105
|
+
// belt-and-suspenders, not the real gate.
|
|
106
|
+
if (verb.continueAfter && options.onToolStep) {
|
|
107
|
+
if (verb.text)
|
|
108
|
+
options.onExplain(verb.text);
|
|
109
|
+
options.onNavigate?.(verb.route);
|
|
110
|
+
// A client-side route change is itself an async re-render (a new
|
|
111
|
+
// page's whole DOM mounting) — same real race waitForDomSettle
|
|
112
|
+
// already closes for fill/click, arguably more likely here. The
|
|
113
|
+
// NEXT resolveVerb call needs the settled new page's context, not
|
|
114
|
+
// whatever was on screen the instant router.push was called.
|
|
115
|
+
void (0, element_ladder_1.waitForDomSettle)(300, 200, 2000).then(() => {
|
|
116
|
+
options.onToolStep?.({ verb: "navigate", target: verb.route, ok: true, observation: `Navigated to ${verb.route}.` });
|
|
117
|
+
});
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
74
120
|
options.onNavigate?.(verb.route);
|
|
75
121
|
if (verb.text)
|
|
76
122
|
options.onExplain(verb.text);
|
|
77
123
|
return;
|
|
124
|
+
}
|
|
78
125
|
case "do": {
|
|
79
126
|
const allowed = options.registeredActions ?? [];
|
|
80
127
|
if (allowed.includes(verb.action)) {
|
|
@@ -101,9 +148,11 @@ function dispatchVerb(verb, route, options) {
|
|
|
101
148
|
// on a different page) — never fired in addition to a real
|
|
102
149
|
// click, so the action can't run twice.
|
|
103
150
|
(0, element_ladder_1.highlightElement)(el);
|
|
104
|
-
el.
|
|
105
|
-
|
|
106
|
-
|
|
151
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
152
|
+
el.click();
|
|
153
|
+
if (verb.text)
|
|
154
|
+
options.onExplain(verb.text);
|
|
155
|
+
});
|
|
107
156
|
return;
|
|
108
157
|
}
|
|
109
158
|
if (verb.target)
|
|
@@ -148,26 +197,48 @@ function dispatchVerb(verb, route, options) {
|
|
|
148
197
|
return;
|
|
149
198
|
}
|
|
150
199
|
(0, element_ladder_1.highlightElement)(el);
|
|
151
|
-
el.
|
|
152
|
-
|
|
200
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
201
|
+
el.click();
|
|
202
|
+
// Real, live-found race this closes — see waitForDomSettle's own
|
|
203
|
+
// doc comment: a click can trigger an async re-render (a cart count
|
|
204
|
+
// updating, a filtered list refreshing) that hasn't happened yet the
|
|
205
|
+
// instant .click() returns. A subsequent read step in the same turn
|
|
206
|
+
// needs the SETTLED result, not whatever was on screen a moment ago.
|
|
207
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
208
|
+
options.onToolStep?.({ verb: "click", target: verb.target, ok: true, observation: "Clicked it." });
|
|
209
|
+
});
|
|
210
|
+
});
|
|
153
211
|
return;
|
|
154
212
|
}
|
|
155
213
|
case "fill": {
|
|
156
214
|
if (verb.text)
|
|
157
215
|
options.onExplain(verb.text);
|
|
158
216
|
const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
159
|
-
if (!el
|
|
217
|
+
if (!el) {
|
|
160
218
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
161
|
-
options.onToolStep?.({
|
|
162
|
-
verb: "fill",
|
|
163
|
-
target: verb.target,
|
|
164
|
-
ok: false,
|
|
165
|
-
observation: el ? "That element isn't a real form field — can't type into it." : "Could not find that element on the page.",
|
|
166
|
-
});
|
|
219
|
+
options.onToolStep?.({ verb: "fill", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
167
220
|
return;
|
|
168
221
|
}
|
|
169
222
|
(0, element_ladder_1.highlightElement)(el);
|
|
170
|
-
|
|
223
|
+
// fillElement itself is both the "is this a real form field" check
|
|
224
|
+
// AND the commit (it sets the value the instant it returns true) —
|
|
225
|
+
// deliberately called from inside this .then(), not the synchronous
|
|
226
|
+
// miss-check above, so the cursor is genuinely seen arriving BEFORE
|
|
227
|
+
// any text appears, not simultaneously with it.
|
|
228
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
229
|
+
if (!(0, element_ladder_1.fillElement)(el, verb.value)) {
|
|
230
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
231
|
+
options.onToolStep?.({ verb: "fill", target: verb.target, ok: false, observation: "That element isn't a real form field — can't type into it." });
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
// See the click case's own comment — the exact real bug found live:
|
|
235
|
+
// typing into a search box, then reading the still-unfiltered
|
|
236
|
+
// results a moment later and reporting a match the real, since-
|
|
237
|
+
// filtered page never actually showed.
|
|
238
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
239
|
+
options.onToolStep?.({ verb: "fill", target: verb.target, ok: true, observation: `Typed "${verb.value}" into it.` });
|
|
240
|
+
});
|
|
241
|
+
});
|
|
171
242
|
return;
|
|
172
243
|
}
|
|
173
244
|
case "read": {
|
|
@@ -179,17 +250,138 @@ function dispatchVerb(verb, route, options) {
|
|
|
179
250
|
options.onToolStep?.({ verb: "read", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
180
251
|
return;
|
|
181
252
|
}
|
|
182
|
-
|
|
253
|
+
// No mutation here, but the cursor still visits what's being read —
|
|
254
|
+
// real visual proof of what the agent is actually looking at, not
|
|
255
|
+
// just a claim in the eventual reported observation.
|
|
256
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
257
|
+
options.onToolStep?.({ verb: "read", target: verb.target, ok: true, observation: (0, element_ladder_1.readElement)(el) });
|
|
258
|
+
});
|
|
183
259
|
return;
|
|
184
260
|
}
|
|
185
261
|
case "call_tool": {
|
|
186
262
|
if (verb.text)
|
|
187
263
|
options.onExplain(verb.text);
|
|
188
|
-
void (0, webmcp_client_1.executeWebMcpTool)(verb.name, verb.args).then((result) => {
|
|
264
|
+
void (0, webmcp_client_1.executeWebMcpTool)(verb.name, verb.args, options.onConfirmTool).then((result) => {
|
|
189
265
|
options.onToolStep?.({ verb: "call_tool", target: verb.name, ok: result.ok, observation: result.observation });
|
|
190
266
|
});
|
|
191
267
|
return;
|
|
192
268
|
}
|
|
269
|
+
case "drag": {
|
|
270
|
+
if (verb.text)
|
|
271
|
+
options.onExplain(verb.text);
|
|
272
|
+
const from = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
273
|
+
const to = from ? (0, element_ladder_1.findElement)(verb.to, options.liveElements) : null;
|
|
274
|
+
if (!from || !to) {
|
|
275
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: from ? verb.to : verb.target, route });
|
|
276
|
+
options.onToolStep?.({ verb: "drag", target: verb.target, ok: false, observation: from ? "Could not find the drop destination on the page." : "Could not find that element on the page." });
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
(0, element_ladder_1.highlightElement)(from);
|
|
280
|
+
void (0, cursor_overlay_1.moveCursorTo)(from).then(() => {
|
|
281
|
+
(0, element_ladder_1.dragElement)(from, to);
|
|
282
|
+
// The cursor also glides to the drop point — fire-and-forget, not
|
|
283
|
+
// awaited, since it's purely a visual echo of a drag that already
|
|
284
|
+
// happened via real pointer events; nothing downstream depends on
|
|
285
|
+
// it finishing.
|
|
286
|
+
void (0, cursor_overlay_1.moveCursorTo)(to);
|
|
287
|
+
// Same real re-render race as click/fill — a drop can trigger an
|
|
288
|
+
// async re-render (a canvas connection line, a reordered list) that
|
|
289
|
+
// hasn't settled the instant the pointer sequence finishes.
|
|
290
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
291
|
+
options.onToolStep?.({ verb: "drag", target: verb.target, ok: true, observation: `Dragged it to ${verb.to}.` });
|
|
292
|
+
});
|
|
293
|
+
});
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
case "select": {
|
|
297
|
+
if (verb.text)
|
|
298
|
+
options.onExplain(verb.text);
|
|
299
|
+
const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
300
|
+
if (!el) {
|
|
301
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
302
|
+
options.onToolStep?.({ verb: "select", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
306
|
+
// Same reasoning as fill's own comment — selectOption is both the
|
|
307
|
+
// "does a matching option exist" check and the commit, so it's called
|
|
308
|
+
// from inside this .then(), after the cursor genuinely arrives.
|
|
309
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
310
|
+
if (!(0, element_ladder_1.selectOption)(el, verb.value)) {
|
|
311
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
312
|
+
options.onToolStep?.({ verb: "select", target: verb.target, ok: false, observation: `Could not find an option matching "${verb.value}".` });
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
316
|
+
options.onToolStep?.({ verb: "select", target: verb.target, ok: true, observation: `Selected "${verb.value}".` });
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
case "key": {
|
|
322
|
+
if (verb.text)
|
|
323
|
+
options.onExplain(verb.text);
|
|
324
|
+
const el = verb.target ? (0, element_ladder_1.findElement)(verb.target, options.liveElements) : document.activeElement;
|
|
325
|
+
if (!el) {
|
|
326
|
+
if (verb.target)
|
|
327
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
328
|
+
options.onToolStep?.({ verb: "key", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
const afterMove = () => {
|
|
332
|
+
(0, element_ladder_1.pressKey)(el, verb.key);
|
|
333
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
334
|
+
options.onToolStep?.({ verb: "key", target: verb.target, ok: true, observation: `Pressed ${verb.key}.` });
|
|
335
|
+
});
|
|
336
|
+
};
|
|
337
|
+
// With no explicit target ("whatever's currently focused"), there's
|
|
338
|
+
// nothing sensible for the cursor to glide to — call straight through
|
|
339
|
+
// instead of routing through a promise callback for no reason, so
|
|
340
|
+
// this path stays exactly as synchronous as it always was.
|
|
341
|
+
if (verb.target) {
|
|
342
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(afterMove);
|
|
343
|
+
}
|
|
344
|
+
else {
|
|
345
|
+
afterMove();
|
|
346
|
+
}
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
case "scroll": {
|
|
350
|
+
if (verb.text)
|
|
351
|
+
options.onExplain(verb.text);
|
|
352
|
+
const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
353
|
+
if (!el) {
|
|
354
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
355
|
+
options.onToolStep?.({ verb: "scroll", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
// A real, already-known element (never a coordinate or something
|
|
359
|
+
// not yet discovered) — highlightElement's own scrollIntoView is
|
|
360
|
+
// exactly the real repositioning this verb exists for; the glow
|
|
361
|
+
// also gives the user a visible cue of where the agent just moved.
|
|
362
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
363
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
364
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
365
|
+
options.onToolStep?.({ verb: "scroll", target: verb.target, ok: true, observation: "Scrolled it into view." });
|
|
366
|
+
});
|
|
367
|
+
});
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
case "wait_for": {
|
|
371
|
+
if (verb.text)
|
|
372
|
+
options.onExplain(verb.text);
|
|
373
|
+
void (0, element_ladder_1.findElementWithRetry)(verb.target, options.liveElements, WAIT_FOR_ATTEMPTS, WAIT_FOR_DELAY_MS).then((el) => {
|
|
374
|
+
if (!el) {
|
|
375
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
376
|
+
options.onToolStep?.({ verb: "wait_for", target: verb.target, ok: false, observation: "It never appeared." });
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
void (0, cursor_overlay_1.moveCursorTo)(el).then(() => {
|
|
380
|
+
options.onToolStep?.({ verb: "wait_for", target: verb.target, ok: true, observation: "It appeared." });
|
|
381
|
+
});
|
|
382
|
+
});
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
193
385
|
// Several click/fill/read/call_tool steps in one round trip instead of
|
|
194
386
|
// one each — server.ts's resolveVerb already validated every action's
|
|
195
387
|
// target/name against real state before this ever arrived. Runs in
|
|
@@ -212,48 +404,132 @@ async function executeBatchActions(actions, route, options) {
|
|
|
212
404
|
const steps = [];
|
|
213
405
|
for (const action of actions) {
|
|
214
406
|
const result = await executeOneBatchAction(action, route, options);
|
|
215
|
-
|
|
407
|
+
const label = ("target" in action && action.target) || ("name" in action && action.name) || "(focused element)";
|
|
408
|
+
steps.push(`${action.verb} ${label}: ${result.observation}`);
|
|
216
409
|
if (!result.ok)
|
|
217
410
|
return { ok: false, observation: steps.join(" | ") };
|
|
218
411
|
}
|
|
219
412
|
return { ok: true, observation: steps.join(" | ") };
|
|
220
413
|
}
|
|
414
|
+
// Phase 3 step 4 — real, bounded, LLM-free retry latitude for the
|
|
415
|
+
// Executor's own lookups (CODA's own point: the Executor stays
|
|
416
|
+
// opinion-free; anything requiring judgment escalates to the Critic,
|
|
417
|
+
// which now genuinely exists as of step 3). Scoped to batch specifically,
|
|
418
|
+
// per the plan's own build order — a batch's later steps are the ones
|
|
419
|
+
// most likely to race a DOM update the batch's OWN earlier step just
|
|
420
|
+
// triggered, which is exactly the "stale re-render" case this recovers
|
|
421
|
+
// from; single-step click/fill/read stay unchanged (findElement, no
|
|
422
|
+
// retry) rather than widening scope beyond what was actually planned.
|
|
221
423
|
async function executeOneBatchAction(action, route, options) {
|
|
222
424
|
switch (action.verb) {
|
|
223
425
|
case "click": {
|
|
224
|
-
const el = (0, element_ladder_1.
|
|
426
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
225
427
|
if (!el) {
|
|
226
428
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
227
429
|
return { ok: false, observation: "Could not find that element on the page." };
|
|
228
430
|
}
|
|
229
431
|
(0, element_ladder_1.highlightElement)(el);
|
|
432
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
230
433
|
el.click();
|
|
434
|
+
// Same real race as the single-step case (see waitForDomSettle's own
|
|
435
|
+
// doc comment) — arguably MORE likely here, since a batch's next
|
|
436
|
+
// step often deliberately reads what THIS step just changed.
|
|
437
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
231
438
|
return { ok: true, observation: "Clicked it." };
|
|
232
439
|
}
|
|
233
440
|
case "fill": {
|
|
234
|
-
const el = (0, element_ladder_1.
|
|
235
|
-
if (!el
|
|
441
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
442
|
+
if (!el) {
|
|
236
443
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
237
|
-
return {
|
|
238
|
-
ok: false,
|
|
239
|
-
observation: el ? "That element isn't a real form field — can't type into it." : "Could not find that element on the page.",
|
|
240
|
-
};
|
|
444
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
241
445
|
}
|
|
242
446
|
(0, element_ladder_1.highlightElement)(el);
|
|
447
|
+
// See the single-step fill case's own comment — fillElement is both
|
|
448
|
+
// the field-type check and the commit, called after the cursor
|
|
449
|
+
// genuinely arrives rather than in the synchronous miss-check.
|
|
450
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
451
|
+
if (!(0, element_ladder_1.fillElement)(el, action.value)) {
|
|
452
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
453
|
+
return { ok: false, observation: "That element isn't a real form field — can't type into it." };
|
|
454
|
+
}
|
|
455
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
243
456
|
return { ok: true, observation: `Typed "${action.value}" into it.` };
|
|
244
457
|
}
|
|
245
458
|
case "read": {
|
|
246
|
-
const el = (0, element_ladder_1.
|
|
459
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
247
460
|
if (!el) {
|
|
248
461
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
249
462
|
return { ok: false, observation: "Could not find that element on the page." };
|
|
250
463
|
}
|
|
464
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
251
465
|
return { ok: true, observation: (0, element_ladder_1.readElement)(el) };
|
|
252
466
|
}
|
|
253
467
|
case "call_tool": {
|
|
254
|
-
const result = await (0, webmcp_client_1.executeWebMcpTool)(action.name, action.args);
|
|
468
|
+
const result = await (0, webmcp_client_1.executeWebMcpTool)(action.name, action.args, options.onConfirmTool);
|
|
255
469
|
return { ok: result.ok, observation: result.observation };
|
|
256
470
|
}
|
|
471
|
+
case "drag": {
|
|
472
|
+
const from = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
473
|
+
const to = from ? await (0, element_ladder_1.findElementWithRetry)(action.to, options.liveElements) : null;
|
|
474
|
+
if (!from || !to) {
|
|
475
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: from ? action.to : action.target, route });
|
|
476
|
+
return { ok: false, observation: from ? "Could not find the drop destination on the page." : "Could not find that element on the page." };
|
|
477
|
+
}
|
|
478
|
+
(0, element_ladder_1.highlightElement)(from);
|
|
479
|
+
await (0, cursor_overlay_1.moveCursorTo)(from);
|
|
480
|
+
(0, element_ladder_1.dragElement)(from, to);
|
|
481
|
+
void (0, cursor_overlay_1.moveCursorTo)(to); // visual echo of the drop point — not awaited, purely decorative
|
|
482
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
483
|
+
return { ok: true, observation: `Dragged it to ${action.to}.` };
|
|
484
|
+
}
|
|
485
|
+
case "select": {
|
|
486
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
487
|
+
if (!el) {
|
|
488
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
489
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
490
|
+
}
|
|
491
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
492
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
493
|
+
if (!(0, element_ladder_1.selectOption)(el, action.value)) {
|
|
494
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
495
|
+
return { ok: false, observation: `Could not find an option matching "${action.value}".` };
|
|
496
|
+
}
|
|
497
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
498
|
+
return { ok: true, observation: `Selected "${action.value}".` };
|
|
499
|
+
}
|
|
500
|
+
case "key": {
|
|
501
|
+
const el = action.target ? await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements) : document.activeElement;
|
|
502
|
+
if (!el) {
|
|
503
|
+
if (action.target)
|
|
504
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
505
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
506
|
+
}
|
|
507
|
+
if (action.target)
|
|
508
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
509
|
+
(0, element_ladder_1.pressKey)(el, action.key);
|
|
510
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
511
|
+
return { ok: true, observation: `Pressed ${action.key}.` };
|
|
512
|
+
}
|
|
513
|
+
case "scroll": {
|
|
514
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
515
|
+
if (!el) {
|
|
516
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
517
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
518
|
+
}
|
|
519
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
520
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
521
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
522
|
+
return { ok: true, observation: "Scrolled it into view." };
|
|
523
|
+
}
|
|
524
|
+
case "wait_for": {
|
|
525
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements, WAIT_FOR_ATTEMPTS, WAIT_FOR_DELAY_MS);
|
|
526
|
+
if (!el) {
|
|
527
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
528
|
+
return { ok: false, observation: "It never appeared." };
|
|
529
|
+
}
|
|
530
|
+
await (0, cursor_overlay_1.moveCursorTo)(el);
|
|
531
|
+
return { ok: true, observation: "It appeared." };
|
|
532
|
+
}
|
|
257
533
|
}
|
|
258
534
|
}
|
|
259
535
|
/**
|
package/dist/webmcp-client.d.ts
CHANGED
|
@@ -6,8 +6,21 @@ export declare function discoverWebMcpTools(): Promise<WebMcpTool[]>;
|
|
|
6
6
|
* exact request's own discoverWebMcpTools() call), never invented.
|
|
7
7
|
* Returns a plain-text observation for the agent loop to reason about
|
|
8
8
|
* next, the same shape a click/fill/read result already takes.
|
|
9
|
+
*
|
|
10
|
+
* Architecture Pillar 6 (the safety layer) — `confirmTool` is only ever
|
|
11
|
+
* consulted for a tool whose OWN registration declared `riskTier:
|
|
12
|
+
* "confirm"` (never something the model or this call site can widen) — a
|
|
13
|
+
* real-world-effect tool (a payment, a delete, anything hard to undo)
|
|
14
|
+
* that must get a genuine yes from the END USER before it runs, not just
|
|
15
|
+
* the model's own decision to call it. No `confirmTool` provided (a host
|
|
16
|
+
* app that hasn't wired up a confirmation UI) is treated as a decline,
|
|
17
|
+
* never as an implicit yes — the safe default when there's no real way
|
|
18
|
+
* to ask.
|
|
9
19
|
*/
|
|
10
|
-
export declare function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined
|
|
20
|
+
export declare function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined, confirmTool?: (tool: {
|
|
21
|
+
name: string;
|
|
22
|
+
description: string;
|
|
23
|
+
}) => Promise<boolean>): Promise<{
|
|
11
24
|
ok: boolean;
|
|
12
25
|
observation: string;
|
|
13
26
|
}>;
|
package/dist/webmcp-client.js
CHANGED
|
@@ -34,6 +34,11 @@ async function discoverWebMcpTools() {
|
|
|
34
34
|
name: String(tool.name),
|
|
35
35
|
description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
|
|
36
36
|
inputSchema: tool.inputSchema,
|
|
37
|
+
// Architecture Pillar 6 — passed through only when the page's own
|
|
38
|
+
// registration declared a real "confirm" tier; anything else
|
|
39
|
+
// (absent, or a value that isn't literally "confirm") stays
|
|
40
|
+
// undefined/"safe" — never invented, never widened by a typo.
|
|
41
|
+
riskTier: tool.riskTier === "confirm" ? "confirm" : undefined,
|
|
37
42
|
}));
|
|
38
43
|
}
|
|
39
44
|
catch {
|
|
@@ -49,8 +54,18 @@ async function discoverWebMcpTools() {
|
|
|
49
54
|
* exact request's own discoverWebMcpTools() call), never invented.
|
|
50
55
|
* Returns a plain-text observation for the agent loop to reason about
|
|
51
56
|
* next, the same shape a click/fill/read result already takes.
|
|
57
|
+
*
|
|
58
|
+
* Architecture Pillar 6 (the safety layer) — `confirmTool` is only ever
|
|
59
|
+
* consulted for a tool whose OWN registration declared `riskTier:
|
|
60
|
+
* "confirm"` (never something the model or this call site can widen) — a
|
|
61
|
+
* real-world-effect tool (a payment, a delete, anything hard to undo)
|
|
62
|
+
* that must get a genuine yes from the END USER before it runs, not just
|
|
63
|
+
* the model's own decision to call it. No `confirmTool` provided (a host
|
|
64
|
+
* app that hasn't wired up a confirmation UI) is treated as a decline,
|
|
65
|
+
* never as an implicit yes — the safe default when there's no real way
|
|
66
|
+
* to ask.
|
|
52
67
|
*/
|
|
53
|
-
async function executeWebMcpTool(name, args) {
|
|
68
|
+
async function executeWebMcpTool(name, args, confirmTool) {
|
|
54
69
|
const modelContext = getModelContext();
|
|
55
70
|
if (!modelContext?.getTools || !modelContext.executeTool) {
|
|
56
71
|
return { ok: false, observation: "This page no longer has that tool available." };
|
|
@@ -60,6 +75,12 @@ async function executeWebMcpTool(name, args) {
|
|
|
60
75
|
const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
|
|
61
76
|
if (!tool)
|
|
62
77
|
return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
|
|
78
|
+
if (tool.riskTier === "confirm") {
|
|
79
|
+
const confirmed = confirmTool ? await confirmTool({ name: tool.name, description: tool.description ?? "" }) : false;
|
|
80
|
+
if (!confirmed) {
|
|
81
|
+
return { ok: false, observation: "This action needs the user's real confirmation before it can run, and it wasn't confirmed." };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
63
84
|
const result = await modelContext.executeTool(tool, args ?? {});
|
|
64
85
|
const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
|
|
65
86
|
return { ok: true, observation: observation.slice(0, 2000) };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cairnvibe/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "In-app AI copilot — <Copilot/> for React/Next.js, <cairn-widget> for any framework — plus the server handlers and realtime voice relay behind them.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": { "access": "public" },
|
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
"./server": "./dist/server.js",
|
|
24
24
|
"./dashboard": "./dist/dashboard.js",
|
|
25
25
|
"./dashboard-sqlite": "./dist/dashboard-sqlite.js",
|
|
26
|
+
"./memory-sqlite": "./dist/memory-sqlite.js",
|
|
27
|
+
"./skill-store": "./dist/skill-store.js",
|
|
26
28
|
"./transcribe-server": "./dist/transcribe-server.js",
|
|
27
29
|
"./speak-server": "./dist/speak-server.js",
|
|
28
30
|
"./realtime-server": "./dist/realtime-server.js",
|