@cairnvibe/sdk 0.2.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-loop.d.ts +113 -0
- package/dist/agent-loop.js +128 -0
- package/dist/cairn-widget.js +2 -2
- package/dist/element-ladder.d.ts +71 -0
- package/dist/element-ladder.js +168 -0
- package/dist/index.d.ts +79 -1
- package/dist/index.js +864 -89
- package/dist/key-rotator.d.ts +28 -0
- package/dist/key-rotator.js +57 -3
- package/dist/memory-sqlite.d.ts +86 -0
- package/dist/memory-sqlite.js +230 -0
- package/dist/realtime-cli.js +22 -1
- package/dist/realtime-server.d.ts +83 -2
- package/dist/realtime-server.js +549 -120
- package/dist/server.d.ts +249 -5
- package/dist/server.js +984 -77
- package/dist/skill-store.d.ts +17 -0
- package/dist/skill-store.js +78 -0
- package/dist/tts-stream.d.ts +25 -0
- package/dist/tts-stream.js +32 -0
- package/dist/vad.d.ts +27 -0
- package/dist/vad.js +128 -0
- package/dist/verb-executor.d.ts +32 -11
- package/dist/verb-executor.js +224 -16
- package/dist/webmcp-client.d.ts +14 -1
- package/dist/webmcp-client.js +22 -1
- package/package.json +3 -1
- package/src/agent-loop.ts +222 -0
- package/src/element-ladder.ts +170 -0
- package/src/index.tsx +914 -93
- package/src/key-rotator.ts +57 -2
- package/src/memory-sqlite.ts +283 -0
- package/src/realtime-cli.ts +24 -1
- package/src/realtime-server.ts +655 -122
- package/src/server.ts +1077 -77
- package/src/skill-store.ts +88 -0
- package/src/tts-stream.ts +30 -0
- package/src/vad.ts +153 -0
- package/src/verb-executor.ts +243 -22
- package/src/web-component.ts +82 -17
- package/src/webmcp-client.ts +30 -2
package/dist/verb-executor.js
CHANGED
|
@@ -11,19 +11,33 @@ exports.executeVerbResponse = executeVerbResponse;
|
|
|
11
11
|
const core_1 = require("@cairnvibe/core");
|
|
12
12
|
const element_ladder_1 = require("./element-ladder");
|
|
13
13
|
const webmcp_client_1 = require("./webmcp-client");
|
|
14
|
+
// wait_for's own real, bounded retry budget — longer than
|
|
15
|
+
// findElementWithRetry's own default (2 attempts, 300ms apart, ~300ms
|
|
16
|
+
// total), since this verb exists specifically for "I know something
|
|
17
|
+
// async should show up" — a toast, a panel appearing after a click — not
|
|
18
|
+
// the incidental transient-miss recovery findElementWithRetry's default
|
|
19
|
+
// already covers for click/fill/batch steps.
|
|
20
|
+
const WAIT_FOR_ATTEMPTS = 6;
|
|
21
|
+
const WAIT_FOR_DELAY_MS = 500;
|
|
14
22
|
/**
|
|
15
23
|
* Promise wrapper around executeVerbResponse for a continuing verb
|
|
16
|
-
* (click/fill/read/call_tool
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
24
|
+
* (click/fill/read/call_tool, or now a navigate marked `continueAfter` —
|
|
25
|
+
* see isTerminalVerb in @cairnvibe/core) — resolves once the real action
|
|
26
|
+
* has actually finished (synchronously for click/fill/read, after a real
|
|
27
|
+
* await for call_tool/navigate) with its real observation, instead of the
|
|
28
|
+
* fire-and-forget callback shape every other verb uses. This is what a
|
|
29
|
+
* loop driver awaits before deciding whether to call the model again.
|
|
30
|
+
* `onNavigate` is only needed for that new navigate-as-continuing-step
|
|
31
|
+
* case — every existing caller that never passes it keeps working
|
|
32
|
+
* unchanged (a continueAfter navigate with no onNavigate here would just
|
|
33
|
+
* never actually move the page; real callers always pass one, same as
|
|
34
|
+
* handleVerb's own options already do for the terminal case).
|
|
21
35
|
*/
|
|
22
|
-
function executeToolStep(raw, route, liveElements) {
|
|
36
|
+
function executeToolStep(raw, route, liveElements, onNavigate, onConfirmTool) {
|
|
23
37
|
return new Promise((resolve) => {
|
|
24
38
|
// executeVerbResponse only ever reaches onToolStep for a genuinely
|
|
25
39
|
// continuing verb — callers are only expected to call this after
|
|
26
|
-
// already confirming (via
|
|
40
|
+
// already confirming (via isTerminalVerb) that the parsed verb is one,
|
|
27
41
|
// so this should always fire; a real timeout (not an immediate
|
|
28
42
|
// microtask — call_tool's own real network round trip needs the time)
|
|
29
43
|
// is the safety net for the case where it somehow doesn't, so a loop
|
|
@@ -32,6 +46,8 @@ function executeToolStep(raw, route, liveElements) {
|
|
|
32
46
|
executeVerbResponse(raw, route, {
|
|
33
47
|
onExplain: () => { },
|
|
34
48
|
liveElements,
|
|
49
|
+
onNavigate,
|
|
50
|
+
onConfirmTool,
|
|
35
51
|
onToolStep: (result) => {
|
|
36
52
|
clearTimeout(timer);
|
|
37
53
|
resolve(result);
|
|
@@ -70,11 +86,36 @@ function dispatchVerb(verb, route, options) {
|
|
|
70
86
|
options.onExplain(verb.text);
|
|
71
87
|
return;
|
|
72
88
|
}
|
|
73
|
-
case "navigate":
|
|
89
|
+
case "navigate": {
|
|
90
|
+
// Real, live-reported gap this closes: navigate used to ALWAYS end
|
|
91
|
+
// the turn the instant it fired, even for a compound goal like "buy
|
|
92
|
+
// earbuds" that needs navigate, then search, then a real report
|
|
93
|
+
// back — see isTerminalVerb's own doc comment in @cairnvibe/core.
|
|
94
|
+
// `options.onToolStep` is only ever set by executeToolStep's own
|
|
95
|
+
// continuing-step wrapper — handleVerb's options never provide it —
|
|
96
|
+
// so this branch can only run when the caller already confirmed
|
|
97
|
+
// (via isTerminalVerb) that this navigate was genuinely marked
|
|
98
|
+
// continueAfter; the defensive `verb.continueAfter` check here is
|
|
99
|
+
// belt-and-suspenders, not the real gate.
|
|
100
|
+
if (verb.continueAfter && options.onToolStep) {
|
|
101
|
+
if (verb.text)
|
|
102
|
+
options.onExplain(verb.text);
|
|
103
|
+
options.onNavigate?.(verb.route);
|
|
104
|
+
// A client-side route change is itself an async re-render (a new
|
|
105
|
+
// page's whole DOM mounting) — same real race waitForDomSettle
|
|
106
|
+
// already closes for fill/click, arguably more likely here. The
|
|
107
|
+
// NEXT resolveVerb call needs the settled new page's context, not
|
|
108
|
+
// whatever was on screen the instant router.push was called.
|
|
109
|
+
void (0, element_ladder_1.waitForDomSettle)(300, 200, 2000).then(() => {
|
|
110
|
+
options.onToolStep?.({ verb: "navigate", target: verb.route, ok: true, observation: `Navigated to ${verb.route}.` });
|
|
111
|
+
});
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
74
114
|
options.onNavigate?.(verb.route);
|
|
75
115
|
if (verb.text)
|
|
76
116
|
options.onExplain(verb.text);
|
|
77
117
|
return;
|
|
118
|
+
}
|
|
78
119
|
case "do": {
|
|
79
120
|
const allowed = options.registeredActions ?? [];
|
|
80
121
|
if (allowed.includes(verb.action)) {
|
|
@@ -149,7 +190,14 @@ function dispatchVerb(verb, route, options) {
|
|
|
149
190
|
}
|
|
150
191
|
(0, element_ladder_1.highlightElement)(el);
|
|
151
192
|
el.click();
|
|
152
|
-
|
|
193
|
+
// Real, live-found race this closes — see waitForDomSettle's own doc
|
|
194
|
+
// comment: a click can trigger an async re-render (a cart count
|
|
195
|
+
// updating, a filtered list refreshing) that hasn't happened yet the
|
|
196
|
+
// instant .click() returns. A subsequent read step in the same turn
|
|
197
|
+
// needs the SETTLED result, not whatever was on screen a moment ago.
|
|
198
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
199
|
+
options.onToolStep?.({ verb: "click", target: verb.target, ok: true, observation: "Clicked it." });
|
|
200
|
+
});
|
|
153
201
|
return;
|
|
154
202
|
}
|
|
155
203
|
case "fill": {
|
|
@@ -167,7 +215,13 @@ function dispatchVerb(verb, route, options) {
|
|
|
167
215
|
return;
|
|
168
216
|
}
|
|
169
217
|
(0, element_ladder_1.highlightElement)(el);
|
|
170
|
-
|
|
218
|
+
// See the click case's own comment — the exact real bug found live:
|
|
219
|
+
// typing into a search box, then reading the still-unfiltered
|
|
220
|
+
// results a moment later and reporting a match the real, since-
|
|
221
|
+
// filtered page never actually showed.
|
|
222
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
223
|
+
options.onToolStep?.({ verb: "fill", target: verb.target, ok: true, observation: `Typed "${verb.value}" into it.` });
|
|
224
|
+
});
|
|
171
225
|
return;
|
|
172
226
|
}
|
|
173
227
|
case "read": {
|
|
@@ -185,11 +239,99 @@ function dispatchVerb(verb, route, options) {
|
|
|
185
239
|
case "call_tool": {
|
|
186
240
|
if (verb.text)
|
|
187
241
|
options.onExplain(verb.text);
|
|
188
|
-
void (0, webmcp_client_1.executeWebMcpTool)(verb.name, verb.args).then((result) => {
|
|
242
|
+
void (0, webmcp_client_1.executeWebMcpTool)(verb.name, verb.args, options.onConfirmTool).then((result) => {
|
|
189
243
|
options.onToolStep?.({ verb: "call_tool", target: verb.name, ok: result.ok, observation: result.observation });
|
|
190
244
|
});
|
|
191
245
|
return;
|
|
192
246
|
}
|
|
247
|
+
case "drag": {
|
|
248
|
+
if (verb.text)
|
|
249
|
+
options.onExplain(verb.text);
|
|
250
|
+
const from = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
251
|
+
const to = from ? (0, element_ladder_1.findElement)(verb.to, options.liveElements) : null;
|
|
252
|
+
if (!from || !to) {
|
|
253
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: from ? verb.to : verb.target, route });
|
|
254
|
+
options.onToolStep?.({ verb: "drag", target: verb.target, ok: false, observation: from ? "Could not find the drop destination on the page." : "Could not find that element on the page." });
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
(0, element_ladder_1.highlightElement)(from);
|
|
258
|
+
(0, element_ladder_1.dragElement)(from, to);
|
|
259
|
+
// Same real re-render race as click/fill — a drop can trigger an
|
|
260
|
+
// async re-render (a canvas connection line, a reordered list) that
|
|
261
|
+
// hasn't settled the instant the pointer sequence finishes.
|
|
262
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
263
|
+
options.onToolStep?.({ verb: "drag", target: verb.target, ok: true, observation: `Dragged it to ${verb.to}.` });
|
|
264
|
+
});
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
case "select": {
|
|
268
|
+
if (verb.text)
|
|
269
|
+
options.onExplain(verb.text);
|
|
270
|
+
const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
271
|
+
if (!el || !(0, element_ladder_1.selectOption)(el, verb.value)) {
|
|
272
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
273
|
+
options.onToolStep?.({
|
|
274
|
+
verb: "select",
|
|
275
|
+
target: verb.target,
|
|
276
|
+
ok: false,
|
|
277
|
+
observation: el ? `Could not find an option matching "${verb.value}".` : "Could not find that element on the page.",
|
|
278
|
+
});
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
282
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
283
|
+
options.onToolStep?.({ verb: "select", target: verb.target, ok: true, observation: `Selected "${verb.value}".` });
|
|
284
|
+
});
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
case "key": {
|
|
288
|
+
if (verb.text)
|
|
289
|
+
options.onExplain(verb.text);
|
|
290
|
+
const el = verb.target ? (0, element_ladder_1.findElement)(verb.target, options.liveElements) : document.activeElement;
|
|
291
|
+
if (!el) {
|
|
292
|
+
if (verb.target)
|
|
293
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
294
|
+
options.onToolStep?.({ verb: "key", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
(0, element_ladder_1.pressKey)(el, verb.key);
|
|
298
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
299
|
+
options.onToolStep?.({ verb: "key", target: verb.target, ok: true, observation: `Pressed ${verb.key}.` });
|
|
300
|
+
});
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
case "scroll": {
|
|
304
|
+
if (verb.text)
|
|
305
|
+
options.onExplain(verb.text);
|
|
306
|
+
const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
|
|
307
|
+
if (!el) {
|
|
308
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
309
|
+
options.onToolStep?.({ verb: "scroll", target: verb.target, ok: false, observation: "Could not find that element on the page." });
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
// A real, already-known element (never a coordinate or something
|
|
313
|
+
// not yet discovered) — highlightElement's own scrollIntoView is
|
|
314
|
+
// exactly the real repositioning this verb exists for; the glow
|
|
315
|
+
// also gives the user a visible cue of where the agent just moved.
|
|
316
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
317
|
+
void (0, element_ladder_1.waitForDomSettle)().then(() => {
|
|
318
|
+
options.onToolStep?.({ verb: "scroll", target: verb.target, ok: true, observation: "Scrolled it into view." });
|
|
319
|
+
});
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
case "wait_for": {
|
|
323
|
+
if (verb.text)
|
|
324
|
+
options.onExplain(verb.text);
|
|
325
|
+
void (0, element_ladder_1.findElementWithRetry)(verb.target, options.liveElements, WAIT_FOR_ATTEMPTS, WAIT_FOR_DELAY_MS).then((el) => {
|
|
326
|
+
if (!el) {
|
|
327
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
|
|
328
|
+
options.onToolStep?.({ verb: "wait_for", target: verb.target, ok: false, observation: "It never appeared." });
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
options.onToolStep?.({ verb: "wait_for", target: verb.target, ok: true, observation: "It appeared." });
|
|
332
|
+
});
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
193
335
|
// Several click/fill/read/call_tool steps in one round trip instead of
|
|
194
336
|
// one each — server.ts's resolveVerb already validated every action's
|
|
195
337
|
// target/name against real state before this ever arrived. Runs in
|
|
@@ -212,26 +354,40 @@ async function executeBatchActions(actions, route, options) {
|
|
|
212
354
|
const steps = [];
|
|
213
355
|
for (const action of actions) {
|
|
214
356
|
const result = await executeOneBatchAction(action, route, options);
|
|
215
|
-
|
|
357
|
+
const label = ("target" in action && action.target) || ("name" in action && action.name) || "(focused element)";
|
|
358
|
+
steps.push(`${action.verb} ${label}: ${result.observation}`);
|
|
216
359
|
if (!result.ok)
|
|
217
360
|
return { ok: false, observation: steps.join(" | ") };
|
|
218
361
|
}
|
|
219
362
|
return { ok: true, observation: steps.join(" | ") };
|
|
220
363
|
}
|
|
364
|
+
// Phase 3 step 4 — real, bounded, LLM-free retry latitude for the
|
|
365
|
+
// Executor's own lookups (CODA's own point: the Executor stays
|
|
366
|
+
// opinion-free; anything requiring judgment escalates to the Critic,
|
|
367
|
+
// which now genuinely exists as of step 3). Scoped to batch specifically,
|
|
368
|
+
// per the plan's own build order — a batch's later steps are the ones
|
|
369
|
+
// most likely to race a DOM update the batch's OWN earlier step just
|
|
370
|
+
// triggered, which is exactly the "stale re-render" case this recovers
|
|
371
|
+
// from; single-step click/fill/read stay unchanged (findElement, no
|
|
372
|
+
// retry) rather than widening scope beyond what was actually planned.
|
|
221
373
|
async function executeOneBatchAction(action, route, options) {
|
|
222
374
|
switch (action.verb) {
|
|
223
375
|
case "click": {
|
|
224
|
-
const el = (0, element_ladder_1.
|
|
376
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
225
377
|
if (!el) {
|
|
226
378
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
227
379
|
return { ok: false, observation: "Could not find that element on the page." };
|
|
228
380
|
}
|
|
229
381
|
(0, element_ladder_1.highlightElement)(el);
|
|
230
382
|
el.click();
|
|
383
|
+
// Same real race as the single-step case (see waitForDomSettle's own
|
|
384
|
+
// doc comment) — arguably MORE likely here, since a batch's next
|
|
385
|
+
// step often deliberately reads what THIS step just changed.
|
|
386
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
231
387
|
return { ok: true, observation: "Clicked it." };
|
|
232
388
|
}
|
|
233
389
|
case "fill": {
|
|
234
|
-
const el = (0, element_ladder_1.
|
|
390
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
235
391
|
if (!el || !(0, element_ladder_1.fillElement)(el, action.value)) {
|
|
236
392
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
237
393
|
return {
|
|
@@ -240,10 +396,11 @@ async function executeOneBatchAction(action, route, options) {
|
|
|
240
396
|
};
|
|
241
397
|
}
|
|
242
398
|
(0, element_ladder_1.highlightElement)(el);
|
|
399
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
243
400
|
return { ok: true, observation: `Typed "${action.value}" into it.` };
|
|
244
401
|
}
|
|
245
402
|
case "read": {
|
|
246
|
-
const el = (0, element_ladder_1.
|
|
403
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
247
404
|
if (!el) {
|
|
248
405
|
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
249
406
|
return { ok: false, observation: "Could not find that element on the page." };
|
|
@@ -251,9 +408,60 @@ async function executeOneBatchAction(action, route, options) {
|
|
|
251
408
|
return { ok: true, observation: (0, element_ladder_1.readElement)(el) };
|
|
252
409
|
}
|
|
253
410
|
case "call_tool": {
|
|
254
|
-
const result = await (0, webmcp_client_1.executeWebMcpTool)(action.name, action.args);
|
|
411
|
+
const result = await (0, webmcp_client_1.executeWebMcpTool)(action.name, action.args, options.onConfirmTool);
|
|
255
412
|
return { ok: result.ok, observation: result.observation };
|
|
256
413
|
}
|
|
414
|
+
case "drag": {
|
|
415
|
+
const from = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
416
|
+
const to = from ? await (0, element_ladder_1.findElementWithRetry)(action.to, options.liveElements) : null;
|
|
417
|
+
if (!from || !to) {
|
|
418
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: from ? action.to : action.target, route });
|
|
419
|
+
return { ok: false, observation: from ? "Could not find the drop destination on the page." : "Could not find that element on the page." };
|
|
420
|
+
}
|
|
421
|
+
(0, element_ladder_1.highlightElement)(from);
|
|
422
|
+
(0, element_ladder_1.dragElement)(from, to);
|
|
423
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
424
|
+
return { ok: true, observation: `Dragged it to ${action.to}.` };
|
|
425
|
+
}
|
|
426
|
+
case "select": {
|
|
427
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
428
|
+
if (!el || !(0, element_ladder_1.selectOption)(el, action.value)) {
|
|
429
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
430
|
+
return { ok: false, observation: el ? `Could not find an option matching "${action.value}".` : "Could not find that element on the page." };
|
|
431
|
+
}
|
|
432
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
433
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
434
|
+
return { ok: true, observation: `Selected "${action.value}".` };
|
|
435
|
+
}
|
|
436
|
+
case "key": {
|
|
437
|
+
const el = action.target ? await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements) : document.activeElement;
|
|
438
|
+
if (!el) {
|
|
439
|
+
if (action.target)
|
|
440
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
441
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
442
|
+
}
|
|
443
|
+
(0, element_ladder_1.pressKey)(el, action.key);
|
|
444
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
445
|
+
return { ok: true, observation: `Pressed ${action.key}.` };
|
|
446
|
+
}
|
|
447
|
+
case "scroll": {
|
|
448
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements);
|
|
449
|
+
if (!el) {
|
|
450
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
451
|
+
return { ok: false, observation: "Could not find that element on the page." };
|
|
452
|
+
}
|
|
453
|
+
(0, element_ladder_1.highlightElement)(el);
|
|
454
|
+
await (0, element_ladder_1.waitForDomSettle)();
|
|
455
|
+
return { ok: true, observation: "Scrolled it into view." };
|
|
456
|
+
}
|
|
457
|
+
case "wait_for": {
|
|
458
|
+
const el = await (0, element_ladder_1.findElementWithRetry)(action.target, options.liveElements, WAIT_FOR_ATTEMPTS, WAIT_FOR_DELAY_MS);
|
|
459
|
+
if (!el) {
|
|
460
|
+
(options.onMiss ?? element_ladder_1.logMiss)({ attempted: action.target, route });
|
|
461
|
+
return { ok: false, observation: "It never appeared." };
|
|
462
|
+
}
|
|
463
|
+
return { ok: true, observation: "It appeared." };
|
|
464
|
+
}
|
|
257
465
|
}
|
|
258
466
|
}
|
|
259
467
|
/**
|
package/dist/webmcp-client.d.ts
CHANGED
|
@@ -6,8 +6,21 @@ export declare function discoverWebMcpTools(): Promise<WebMcpTool[]>;
|
|
|
6
6
|
* exact request's own discoverWebMcpTools() call), never invented.
|
|
7
7
|
* Returns a plain-text observation for the agent loop to reason about
|
|
8
8
|
* next, the same shape a click/fill/read result already takes.
|
|
9
|
+
*
|
|
10
|
+
* Architecture Pillar 6 (the safety layer) — `confirmTool` is only ever
|
|
11
|
+
* consulted for a tool whose OWN registration declared `riskTier:
|
|
12
|
+
* "confirm"` (never something the model or this call site can widen) — a
|
|
13
|
+
* real-world-effect tool (a payment, a delete, anything hard to undo)
|
|
14
|
+
* that must get a genuine yes from the END USER before it runs, not just
|
|
15
|
+
* the model's own decision to call it. No `confirmTool` provided (a host
|
|
16
|
+
* app that hasn't wired up a confirmation UI) is treated as a decline,
|
|
17
|
+
* never as an implicit yes — the safe default when there's no real way
|
|
18
|
+
* to ask.
|
|
9
19
|
*/
|
|
10
|
-
export declare function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined
|
|
20
|
+
export declare function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined, confirmTool?: (tool: {
|
|
21
|
+
name: string;
|
|
22
|
+
description: string;
|
|
23
|
+
}) => Promise<boolean>): Promise<{
|
|
11
24
|
ok: boolean;
|
|
12
25
|
observation: string;
|
|
13
26
|
}>;
|
package/dist/webmcp-client.js
CHANGED
|
@@ -34,6 +34,11 @@ async function discoverWebMcpTools() {
|
|
|
34
34
|
name: String(tool.name),
|
|
35
35
|
description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
|
|
36
36
|
inputSchema: tool.inputSchema,
|
|
37
|
+
// Architecture Pillar 6 — passed through only when the page's own
|
|
38
|
+
// registration declared a real "confirm" tier; anything else
|
|
39
|
+
// (absent, or a value that isn't literally "confirm") stays
|
|
40
|
+
// undefined/"safe" — never invented, never widened by a typo.
|
|
41
|
+
riskTier: tool.riskTier === "confirm" ? "confirm" : undefined,
|
|
37
42
|
}));
|
|
38
43
|
}
|
|
39
44
|
catch {
|
|
@@ -49,8 +54,18 @@ async function discoverWebMcpTools() {
|
|
|
49
54
|
* exact request's own discoverWebMcpTools() call), never invented.
|
|
50
55
|
* Returns a plain-text observation for the agent loop to reason about
|
|
51
56
|
* next, the same shape a click/fill/read result already takes.
|
|
57
|
+
*
|
|
58
|
+
* Architecture Pillar 6 (the safety layer) — `confirmTool` is only ever
|
|
59
|
+
* consulted for a tool whose OWN registration declared `riskTier:
|
|
60
|
+
* "confirm"` (never something the model or this call site can widen) — a
|
|
61
|
+
* real-world-effect tool (a payment, a delete, anything hard to undo)
|
|
62
|
+
* that must get a genuine yes from the END USER before it runs, not just
|
|
63
|
+
* the model's own decision to call it. No `confirmTool` provided (a host
|
|
64
|
+
* app that hasn't wired up a confirmation UI) is treated as a decline,
|
|
65
|
+
* never as an implicit yes — the safe default when there's no real way
|
|
66
|
+
* to ask.
|
|
52
67
|
*/
|
|
53
|
-
async function executeWebMcpTool(name, args) {
|
|
68
|
+
async function executeWebMcpTool(name, args, confirmTool) {
|
|
54
69
|
const modelContext = getModelContext();
|
|
55
70
|
if (!modelContext?.getTools || !modelContext.executeTool) {
|
|
56
71
|
return { ok: false, observation: "This page no longer has that tool available." };
|
|
@@ -60,6 +75,12 @@ async function executeWebMcpTool(name, args) {
|
|
|
60
75
|
const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
|
|
61
76
|
if (!tool)
|
|
62
77
|
return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
|
|
78
|
+
if (tool.riskTier === "confirm") {
|
|
79
|
+
const confirmed = confirmTool ? await confirmTool({ name: tool.name, description: tool.description ?? "" }) : false;
|
|
80
|
+
if (!confirmed) {
|
|
81
|
+
return { ok: false, observation: "This action needs the user's real confirmation before it can run, and it wasn't confirmed." };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
63
84
|
const result = await modelContext.executeTool(tool, args ?? {});
|
|
64
85
|
const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
|
|
65
86
|
return { ok: true, observation: observation.slice(0, 2000) };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cairnvibe/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "In-app AI copilot — <Copilot/> for React/Next.js, <cairn-widget> for any framework — plus the server handlers and realtime voice relay behind them.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": { "access": "public" },
|
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
"./server": "./dist/server.js",
|
|
24
24
|
"./dashboard": "./dist/dashboard.js",
|
|
25
25
|
"./dashboard-sqlite": "./dist/dashboard-sqlite.js",
|
|
26
|
+
"./memory-sqlite": "./dist/memory-sqlite.js",
|
|
27
|
+
"./skill-store": "./dist/skill-store.js",
|
|
26
28
|
"./transcribe-server": "./dist/transcribe-server.js",
|
|
27
29
|
"./speak-server": "./dist/speak-server.js",
|
|
28
30
|
"./realtime-server": "./dist/realtime-server.js",
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
// The shared skeleton behind both agent-loop drivers — index.tsx's
|
|
2
|
+
// runTypedAgentLoop (HTTP/typed transport) and realtime-server.ts's
|
|
3
|
+
// finalizeTurn (WebSocket/voice relay) independently re-implemented the
|
|
4
|
+
// exact same "ask, check terminal, execute a continuing step for real,
|
|
5
|
+
// fold the result into working history, ask again, up to a hard
|
|
6
|
+
// iteration cap" shape — a real, live duplication risk (any future fix
|
|
7
|
+
// to one had to be remembered and re-applied to the other by hand).
|
|
8
|
+
// This module is the first step of the Phase 3 multi-agent redesign
|
|
9
|
+
// (see DEVELOPMENT.md/the plan file's "Phase 3" entry): extract exactly
|
|
10
|
+
// that shared shape, with ZERO behavior change, so Planner/Critic
|
|
11
|
+
// wiring in later steps has one real place to attach to instead of two.
|
|
12
|
+
//
|
|
13
|
+
// Deliberately does NOT own transport-specific side effects — sending a
|
|
14
|
+
// message to a client, speaking, committing to a connection's real
|
|
15
|
+
// cross-turn memory, barge-in cancellation timing. Those stay in each
|
|
16
|
+
// transport's own getNextStep/onStep/onStepResult/executeStep closures,
|
|
17
|
+
// and in what the caller does with this function's return value, exactly
|
|
18
|
+
// as before this extraction. Plain TypeScript only (no JSX, no Node
|
|
19
|
+
// built-ins) — imported as raw source by index.tsx's browser bundle AND
|
|
20
|
+
// compiled to dist/ for realtime-server.ts's Node build.
|
|
21
|
+
|
|
22
|
+
import { isTerminalVerb, type AgentEvent, type CriticVerdict, type HistoryTurn, type VerbResponse } from "@cairnvibe/core";
|
|
23
|
+
|
|
24
|
+
/** 4 exchanges — matches the cap both original drivers independently used. */
|
|
25
|
+
export const MAX_HISTORY_TURNS = 8;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Architecture Pillar 4 — a cheap, LOCAL signal for "this goal probably
|
|
29
|
+
* needs more than one real step," checked BEFORE the first step even
|
|
30
|
+
* runs, so a caller can start the Planner call in PARALLEL with the
|
|
31
|
+
* first getNextStep instead of only after that first step already came
|
|
32
|
+
* back non-terminal (the "lazy gate" the plan singles out for
|
|
33
|
+
* replacement — realtime-server.ts's own onStep used to build planPromise
|
|
34
|
+
* only once `!terminal && iteration === 0` was already true, one full
|
|
35
|
+
* model round trip later than it needed to be). Deliberately
|
|
36
|
+
* conservative, on purpose: a false negative here just falls back to
|
|
37
|
+
* that same lazy-after-step-1 behavior — unchanged, zero regression —
|
|
38
|
+
* while a false positive costs one Planner call that would have started
|
|
39
|
+
* a moment later anyway, never a wrong answer. Genuine UI-pattern-aware
|
|
40
|
+
* classification (Pillar 2, not built yet) can replace this heuristic
|
|
41
|
+
* later without changing what calls it. Lives here (not server.ts) so
|
|
42
|
+
* BOTH transports can use the exact same check: this file is plain,
|
|
43
|
+
* dependency-free TypeScript imported as raw source by index.tsx's
|
|
44
|
+
* browser bundle AND compiled for realtime-server.ts's Node build — a
|
|
45
|
+
* server-only file (server.ts imports the Anthropic/Groq SDKs) can never
|
|
46
|
+
* be imported from the client widget.
|
|
47
|
+
*/
|
|
48
|
+
const MULTI_STEP_SIGNAL = /\b(then|after that|once (you|it|that|i)|and then|next,|first[,.]? .*\bthen\b)\b/;
|
|
49
|
+
export function looksMultiStep(question: string): boolean {
|
|
50
|
+
return MULTI_STEP_SIGNAL.test(question.toLowerCase());
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function summarizeVerbForHistory(verb: VerbResponse): string {
|
|
54
|
+
if ("text" in verb && verb.text) return verb.text;
|
|
55
|
+
switch (verb.verb) {
|
|
56
|
+
case "highlight":
|
|
57
|
+
case "open":
|
|
58
|
+
return `(highlighted ${verb.target})`;
|
|
59
|
+
case "navigate":
|
|
60
|
+
return `(navigated to ${verb.route})`;
|
|
61
|
+
case "do":
|
|
62
|
+
return `(ran ${verb.action}${verb.target ? ` on ${verb.target}` : ""})`;
|
|
63
|
+
case "tour":
|
|
64
|
+
return verb.steps.map((s) => s.text).join(" ");
|
|
65
|
+
case "click":
|
|
66
|
+
return `(clicked ${verb.target})`;
|
|
67
|
+
case "fill":
|
|
68
|
+
return `(typed "${verb.value}" into ${verb.target})`;
|
|
69
|
+
case "read":
|
|
70
|
+
return `(read ${verb.target})`;
|
|
71
|
+
case "call_tool":
|
|
72
|
+
return `(called ${verb.name})`;
|
|
73
|
+
case "drag":
|
|
74
|
+
return `(dragged ${verb.target} to ${verb.to})`;
|
|
75
|
+
case "select":
|
|
76
|
+
return `(selected "${verb.value}" in ${verb.target})`;
|
|
77
|
+
case "key":
|
|
78
|
+
return `(pressed ${verb.key}${verb.target ? ` on ${verb.target}` : ""})`;
|
|
79
|
+
case "batch":
|
|
80
|
+
return `(${verb.actions.length} steps: ${verb.actions.map((a) => a.verb).join(", ")})`;
|
|
81
|
+
default:
|
|
82
|
+
return "(no response)";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface AgentLoopStepEvent {
|
|
87
|
+
verb: VerbResponse;
|
|
88
|
+
/** 0-based. */
|
|
89
|
+
iteration: number;
|
|
90
|
+
/** True if isTerminalVerb(verb) says this ends the loop right after
|
|
91
|
+
* this hook returns (TERMINAL_VERBS membership, except a navigate
|
|
92
|
+
* marked continueAfter — see isTerminalVerb's own doc comment). Lets a
|
|
93
|
+
* caller act differently for a continuing vs. final step without
|
|
94
|
+
* re-deriving that check itself. */
|
|
95
|
+
terminal: boolean;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface AgentLoopStepResultEvent {
|
|
99
|
+
verb: VerbResponse;
|
|
100
|
+
iteration: number;
|
|
101
|
+
observation: string | null | undefined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export interface AgentLoopDeps {
|
|
105
|
+
/** Resolve the next step given the CURRENT working history. Return
|
|
106
|
+
* `null` for a response that failed to parse/validate — the HTTP
|
|
107
|
+
* path's own real case (a raw fetch response might not conform);
|
|
108
|
+
* realtime's in-process resolveVerb never produces this, since it
|
|
109
|
+
* always returns a valid VerbResponse itself. A null return ends the
|
|
110
|
+
* loop immediately with outcome "unparseable". */
|
|
111
|
+
getNextStep(loopHistory: HistoryTurn[], iteration: number): Promise<VerbResponse | null>;
|
|
112
|
+
/** Fires immediately after getNextStep resolves, before the terminal/
|
|
113
|
+
* continuing branch is decided — the real-time side-effect point (send
|
|
114
|
+
* the verb to a client, trigger an ack on the first continuing step,
|
|
115
|
+
* check for a superseding barge-in). Returning true aborts the loop
|
|
116
|
+
* immediately: no further side effects, outcome "aborted". */
|
|
117
|
+
onStep?(event: AgentLoopStepEvent): boolean | Promise<boolean>;
|
|
118
|
+
/** Execute a continuing verb (click/fill/read/call_tool/batch) for
|
|
119
|
+
* real; return its observation text (or null/undefined for "no
|
|
120
|
+
* result", folded into history as "no result" exactly like both
|
|
121
|
+
* original drivers already did). */
|
|
122
|
+
executeStep(verb: VerbResponse, iteration: number): Promise<string | null | undefined>;
|
|
123
|
+
/** Fires after executeStep resolves, before folding the observation
|
|
124
|
+
* into working history — a second real-time abort checkpoint (e.g. a
|
|
125
|
+
* barge-in generation check after awaiting a real tool result, which
|
|
126
|
+
* can itself take a while). Returning true aborts the loop with
|
|
127
|
+
* outcome "aborted", discarding this step's observation. */
|
|
128
|
+
onStepResult?(event: AgentLoopStepResultEvent): boolean | Promise<boolean>;
|
|
129
|
+
/**
|
|
130
|
+
* Phase 3 step 3 — a genuinely separate pass over the step's REAL
|
|
131
|
+
* observation, decoupled from the Executor/model's own self-report
|
|
132
|
+
* (the direct fix for the diagnosed bug: a batch succeeded and the
|
|
133
|
+
* model kept looping instead of recognizing it). Fires after
|
|
134
|
+
* onStepResult/the history fold. Returning a "task_complete" or
|
|
135
|
+
* "give_up" verdict ends the loop right here — even though the
|
|
136
|
+
* model's own verb was never a TERMINAL_VERBS member — instead of
|
|
137
|
+
* asking the model again and hoping it notices. Returning "continue"
|
|
138
|
+
* (including after the caller's own closure has silently handled a
|
|
139
|
+
* "replan" by fetching a fresh Plan — driveAgentLoop itself has no
|
|
140
|
+
* concept of a Plan, only of "keep going or stop") keeps the loop
|
|
141
|
+
* going exactly as if this hook were absent. Returning null/undefined
|
|
142
|
+
* behaves the same as "continue" — a caller can choose not to run the
|
|
143
|
+
* Critic on a particular step without a special no-op verdict shape.
|
|
144
|
+
*/
|
|
145
|
+
runCritic?(event: AgentLoopStepResultEvent): Promise<CriticVerdict | null | undefined>;
|
|
146
|
+
/**
|
|
147
|
+
* Phase 3 step 5 — a pure, fire-and-forget event consumer for a
|
|
148
|
+
* Talker-style narration layer ("Revisable by Design"'s pattern):
|
|
149
|
+
* never awaited, never able to affect control flow. driveAgentLoop
|
|
150
|
+
* itself emits "act" (right after a step's onStep/abort check passes —
|
|
151
|
+
* only for a verb that's actually going to execute, never a discarded
|
|
152
|
+
* one) and "obs" (right after onStepResult's own abort check passes),
|
|
153
|
+
* since it already has that data at exactly those points. A caller's
|
|
154
|
+
* own onStep/runCritic closures can call this SAME callback directly —
|
|
155
|
+
* it's just a plain reference they already have via the deps object
|
|
156
|
+
* they constructed — to emit "thk" (Critic reasoning) or "inj"
|
|
157
|
+
* (injected filler narration, e.g. a Talker ack phrase) events too;
|
|
158
|
+
* driveAgentLoop has no opinion on those.
|
|
159
|
+
*/
|
|
160
|
+
onEvent?(event: AgentEvent): void;
|
|
161
|
+
/** Defaults to 6 — a hard cap, not a target, matching both original drivers. */
|
|
162
|
+
maxIterations?: number;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export type AgentLoopOutcome =
|
|
166
|
+
| { outcome: "terminal"; finalVerb: VerbResponse; workingHistory: HistoryTurn[] }
|
|
167
|
+
/** The Critic independently confirmed the (last) task's doneContract
|
|
168
|
+
* is satisfied — the real fix for the diagnosed bug. The caller
|
|
169
|
+
* synthesizes its own terminal-shaped response (e.g. `{verb: "explain",
|
|
170
|
+
* text: verdict.reasoning}`) from `verdict`, same as it would for a
|
|
171
|
+
* model-produced terminal verb. */
|
|
172
|
+
| { outcome: "critic-complete"; verdict: CriticVerdict; workingHistory: HistoryTurn[] }
|
|
173
|
+
/** The Critic (or the harness's own stall-count fail-safe, inside the
|
|
174
|
+
* caller's runCritic closure) decided continuing wouldn't help. */
|
|
175
|
+
| { outcome: "critic-give-up"; verdict: CriticVerdict; workingHistory: HistoryTurn[] }
|
|
176
|
+
| { outcome: "unparseable"; workingHistory: HistoryTurn[] }
|
|
177
|
+
| { outcome: "gave-up"; workingHistory: HistoryTurn[] }
|
|
178
|
+
| { outcome: "aborted"; workingHistory: HistoryTurn[] };
|
|
179
|
+
|
|
180
|
+
export async function driveAgentLoop(initialHistory: HistoryTurn[], deps: AgentLoopDeps): Promise<AgentLoopOutcome> {
|
|
181
|
+
const maxIterations = deps.maxIterations ?? 6;
|
|
182
|
+
let loopHistory = initialHistory;
|
|
183
|
+
|
|
184
|
+
for (let i = 0; i < maxIterations; i++) {
|
|
185
|
+
const verb = await deps.getNextStep(loopHistory, i);
|
|
186
|
+
if (verb === null) return { outcome: "unparseable", workingHistory: loopHistory };
|
|
187
|
+
|
|
188
|
+
const terminal = isTerminalVerb(verb);
|
|
189
|
+
if (deps.onStep) {
|
|
190
|
+
const abort = await deps.onStep({ verb, iteration: i, terminal });
|
|
191
|
+
if (abort) return { outcome: "aborted", workingHistory: loopHistory };
|
|
192
|
+
}
|
|
193
|
+
deps.onEvent?.({ type: "act", verb, at: Date.now() });
|
|
194
|
+
|
|
195
|
+
if (terminal) {
|
|
196
|
+
return { outcome: "terminal", finalVerb: verb, workingHistory: loopHistory };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const observation = await deps.executeStep(verb, i);
|
|
200
|
+
if (deps.onStepResult) {
|
|
201
|
+
const abort = await deps.onStepResult({ verb, iteration: i, observation });
|
|
202
|
+
if (abort) return { outcome: "aborted", workingHistory: loopHistory };
|
|
203
|
+
}
|
|
204
|
+
deps.onEvent?.({ type: "obs", observation: observation ?? "no result", ok: observation !== null && observation !== undefined, at: Date.now() });
|
|
205
|
+
|
|
206
|
+
loopHistory = [
|
|
207
|
+
...loopHistory,
|
|
208
|
+
{ role: "assistant" as const, text: `${summarizeVerbForHistory(verb)}. Result: ${observation ?? "no result"}` },
|
|
209
|
+
].slice(-MAX_HISTORY_TURNS);
|
|
210
|
+
|
|
211
|
+
if (deps.runCritic) {
|
|
212
|
+
const verdict = await deps.runCritic({ verb, iteration: i, observation });
|
|
213
|
+
if (verdict?.verdict === "task_complete") return { outcome: "critic-complete", verdict, workingHistory: loopHistory };
|
|
214
|
+
if (verdict?.verdict === "give_up") return { outcome: "critic-give-up", verdict, workingHistory: loopHistory };
|
|
215
|
+
// "continue", "replan" (already handled inside the caller's own
|
|
216
|
+
// runCritic closure — see this field's own doc comment), or no
|
|
217
|
+
// verdict at all: fall through and keep looping, unchanged.
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
return { outcome: "gave-up", workingHistory: loopHistory };
|
|
222
|
+
}
|