cbrowser 18.67.0 → 18.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/accessibility-empathy.d.ts.map +1 -1
- package/dist/analysis/accessibility-empathy.js +62 -12
- package/dist/analysis/accessibility-empathy.js.map +1 -1
- package/dist/analysis/agent-ready-audit.d.ts.map +1 -1
- package/dist/analysis/agent-ready-audit.js +29 -35
- package/dist/analysis/agent-ready-audit.js.map +1 -1
- package/dist/browser.d.ts +32 -0
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +194 -22
- package/dist/browser.js.map +1 -1
- package/dist/cif-score.js +6 -6
- package/dist/cif-score.js.map +1 -1
- package/dist/cognitive/emotions.d.ts +31 -0
- package/dist/cognitive/emotions.d.ts.map +1 -1
- package/dist/cognitive/emotions.js +129 -0
- package/dist/cognitive/emotions.js.map +1 -1
- package/dist/cognitive/index.d.ts +76 -0
- package/dist/cognitive/index.d.ts.map +1 -1
- package/dist/cognitive/index.js +1380 -126
- package/dist/cognitive/index.js.map +1 -1
- package/dist/cognitive/temporal.d.ts +80 -0
- package/dist/cognitive/temporal.d.ts.map +1 -0
- package/dist/cognitive/temporal.js +179 -0
- package/dist/cognitive/temporal.js.map +1 -0
- package/dist/config.d.ts +24 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js.map +1 -1
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +18 -1
- package/dist/daemon.js.map +1 -1
- package/dist/visual/attention-transport.d.ts +12 -0
- package/dist/visual/attention-transport.d.ts.map +1 -1
- package/dist/visual/attention-transport.js +95 -0
- package/dist/visual/attention-transport.js.map +1 -1
- package/dist/visual/cognitive-transport-chain.d.ts.map +1 -1
- package/dist/visual/cognitive-transport-chain.js +15 -4
- package/dist/visual/cognitive-transport-chain.js.map +1 -1
- package/dist/visual/human-input.d.ts +72 -0
- package/dist/visual/human-input.d.ts.map +1 -0
- package/dist/visual/human-input.js +164 -0
- package/dist/visual/human-input.js.map +1 -0
- package/dist/visual/perceptual-transport.d.ts.map +1 -1
- package/dist/visual/perceptual-transport.js +122 -1
- package/dist/visual/perceptual-transport.js.map +1 -1
- package/package.json +1 -1
package/dist/cognitive/index.js
CHANGED
|
@@ -26,7 +26,7 @@ import { existsSync, readFileSync, writeFileSync } from "fs";
|
|
|
26
26
|
import { join } from "path";
|
|
27
27
|
import { CBrowser } from "../browser.js";
|
|
28
28
|
import { getPersona, getCognitiveProfile, createCognitivePersona, } from "../personas.js";
|
|
29
|
-
import { createInitialEmotionalState, createEmotionalConfig, applyEmotionalTrigger,
|
|
29
|
+
import { createInitialEmotionalState, createEmotionalConfig, applyEmotionalTrigger, decayEmotionsWithTraits, deriveSimplifiedState, calculateAbandonmentModifier, describeEmotionalState, shouldConsiderAbandonment, } from "./emotions.js";
|
|
30
30
|
import { calculateFatigueIncrement, calculateFittsMovementTime, shouldSwitchToSystem2, canReturnToSystem1, calculateTypingTime, calculateScanWidthMultiplier, calculateGazeMouseLag, calculatePeripheralVision, createHabituationState, updateHabituationState, classifyUIPattern, } from "../types.js";
|
|
31
31
|
import { loadConfigFile, getDataDir } from "../config.js";
|
|
32
32
|
// ============================================================================
|
|
@@ -113,6 +113,545 @@ export function isClaudeCodeSession() {
|
|
|
113
113
|
export function isCognitiveAvailable() {
|
|
114
114
|
return isApiKeyConfigured() || isClaudeCodeSession();
|
|
115
115
|
}
|
|
116
|
+
// ============================================================================
|
|
117
|
+
// Cognitive Journey Runner
|
|
118
|
+
// ============================================================================
|
|
119
|
+
/**
|
|
120
|
+
* In-page mouse overlay — injected on every navigation when `enableMouseOverlay`
|
|
121
|
+
* is set. Renders a synthetic cursor + click ripples + action banner inside the
|
|
122
|
+
* page DOM so the screen recording / VNC stream actually shows where the bot
|
|
123
|
+
* is interacting. CDP-driven clicks don't move the OS cursor, so without this
|
|
124
|
+
* the recording looks like things click themselves.
|
|
125
|
+
*
|
|
126
|
+
* Exposes `window.__cbShowAction(text)` for the journey runner to surface what
|
|
127
|
+
* is about to happen on screen.
|
|
128
|
+
*/
|
|
129
|
+
const MOUSE_OVERLAY_SCRIPT = `(() => {
|
|
130
|
+
if (window.__cbHelperInstalled || window.__cbHelperPending) return;
|
|
131
|
+
// addInitScript runs at "earliest" — documentElement may not exist yet, so
|
|
132
|
+
// defer real setup until DOMContentLoaded. Mark "pending" to prevent
|
|
133
|
+
// duplicate scheduling if the script gets re-evaluated mid-load.
|
|
134
|
+
window.__cbHelperPending = true;
|
|
135
|
+
|
|
136
|
+
const install = () => {
|
|
137
|
+
if (window.__cbHelperInstalled) return;
|
|
138
|
+
if (!document.documentElement || !document.body) return;
|
|
139
|
+
window.__cbHelperInstalled = true;
|
|
140
|
+
document.documentElement.setAttribute('data-cb-helper', '1');
|
|
141
|
+
|
|
142
|
+
const css = \`
|
|
143
|
+
/* Hide native OS / chromium cursors entirely so our injected pointer is
|
|
144
|
+
the ONLY one visible. Combined with x11vnc -nocursor on the server side,
|
|
145
|
+
this guarantees a single cursor in VNC captures. */
|
|
146
|
+
html, body, *, *::before, *::after { cursor: none !important; }
|
|
147
|
+
|
|
148
|
+
.cb-cursor {
|
|
149
|
+
position: fixed; left: 0; top: 0;
|
|
150
|
+
width: 56px; height: 56px;
|
|
151
|
+
pointer-events: none;
|
|
152
|
+
z-index: 2147483647;
|
|
153
|
+
transform: translate(-200px, -200px);
|
|
154
|
+
/* GPU-promoted layer — the cursor is updated every frame, so isolating
|
|
155
|
+
it from the rest of the page avoids triggering layout/paint elsewhere
|
|
156
|
+
and gives consistent 60fps motion. */
|
|
157
|
+
will-change: transform;
|
|
158
|
+
transform-origin: 0 0;
|
|
159
|
+
}
|
|
160
|
+
.cb-cursor svg {
|
|
161
|
+
width: 56px; height: 56px;
|
|
162
|
+
filter:
|
|
163
|
+
drop-shadow(0 0 2px #000)
|
|
164
|
+
drop-shadow(0 3px 8px rgba(0,0,0,0.65));
|
|
165
|
+
}
|
|
166
|
+
.cb-wake {
|
|
167
|
+
position: fixed;
|
|
168
|
+
width: 18px; height: 18px;
|
|
169
|
+
border-radius: 50%;
|
|
170
|
+
pointer-events: none;
|
|
171
|
+
z-index: 2147483646;
|
|
172
|
+
transform: translate(-50%, -50%);
|
|
173
|
+
will-change: transform, opacity;
|
|
174
|
+
}
|
|
175
|
+
.cb-wake-1 {
|
|
176
|
+
background: radial-gradient(circle, rgba(99,102,241,0.55) 0%, rgba(99,102,241,0.0) 60%);
|
|
177
|
+
animation: cb-wake-out 0.85s cubic-bezier(0.2,0.8,0.4,1) forwards;
|
|
178
|
+
}
|
|
179
|
+
.cb-wake-2 {
|
|
180
|
+
border: 2px solid rgba(99,102,241,0.95);
|
|
181
|
+
animation: cb-wake-ring 0.9s cubic-bezier(0.2,0.8,0.4,1) forwards;
|
|
182
|
+
}
|
|
183
|
+
@keyframes cb-wake-out {
|
|
184
|
+
0% { transform: translate(-50%, -50%) scale(0.4); opacity: 0.95; }
|
|
185
|
+
100% { transform: translate(-50%, -50%) scale(4.5); opacity: 0; }
|
|
186
|
+
}
|
|
187
|
+
@keyframes cb-wake-ring {
|
|
188
|
+
0% { transform: translate(-50%, -50%) scale(0.5); opacity: 1; border-width: 2px; }
|
|
189
|
+
60% { opacity: 0.7; border-width: 2px; }
|
|
190
|
+
100% { transform: translate(-50%, -50%) scale(5); opacity: 0; border-width: 1px; }
|
|
191
|
+
}
|
|
192
|
+
.cb-scroll-indicator {
|
|
193
|
+
position: fixed;
|
|
194
|
+
right: 24px;
|
|
195
|
+
top: 50%;
|
|
196
|
+
transform: translateY(-50%);
|
|
197
|
+
width: 44px;
|
|
198
|
+
padding: 10px 0;
|
|
199
|
+
background: rgba(15,23,42,0.85);
|
|
200
|
+
color: white;
|
|
201
|
+
border-radius: 22px;
|
|
202
|
+
pointer-events: none;
|
|
203
|
+
z-index: 2147483645;
|
|
204
|
+
display: flex;
|
|
205
|
+
flex-direction: column;
|
|
206
|
+
align-items: center;
|
|
207
|
+
gap: 6px;
|
|
208
|
+
opacity: 0;
|
|
209
|
+
transition: opacity 0.25s;
|
|
210
|
+
box-shadow: 0 4px 16px rgba(0,0,0,0.4);
|
|
211
|
+
}
|
|
212
|
+
.cb-scroll-indicator.cb-show { opacity: 1; }
|
|
213
|
+
.cb-scroll-arrow {
|
|
214
|
+
width: 18px; height: 18px;
|
|
215
|
+
animation: cb-scroll-bob 0.55s ease-in-out infinite alternate;
|
|
216
|
+
}
|
|
217
|
+
.cb-scroll-up .cb-scroll-arrow { transform: rotate(180deg); }
|
|
218
|
+
@keyframes cb-scroll-bob {
|
|
219
|
+
0% { transform: translateY(-3px); opacity: 0.5; }
|
|
220
|
+
100% { transform: translateY(3px); opacity: 1; }
|
|
221
|
+
}
|
|
222
|
+
.cb-action-banner {
|
|
223
|
+
position: fixed;
|
|
224
|
+
top: 14px; left: 50%;
|
|
225
|
+
transform: translateX(-50%);
|
|
226
|
+
padding: 7px 16px;
|
|
227
|
+
max-width: 80vw;
|
|
228
|
+
background: rgba(15,23,42,0.92);
|
|
229
|
+
color: #fff;
|
|
230
|
+
font: 500 13px/1.3 -apple-system, system-ui, sans-serif;
|
|
231
|
+
border-radius: 999px;
|
|
232
|
+
pointer-events: none;
|
|
233
|
+
z-index: 2147483645;
|
|
234
|
+
box-shadow: 0 6px 20px rgba(0,0,0,0.35);
|
|
235
|
+
transition: opacity 0.25s;
|
|
236
|
+
white-space: nowrap;
|
|
237
|
+
overflow: hidden;
|
|
238
|
+
text-overflow: ellipsis;
|
|
239
|
+
opacity: 0;
|
|
240
|
+
}
|
|
241
|
+
.cb-action-banner.cb-show { opacity: 1; }
|
|
242
|
+
\`;
|
|
243
|
+
const style = document.createElement('style');
|
|
244
|
+
style.textContent = css;
|
|
245
|
+
(document.head || document.documentElement).appendChild(style);
|
|
246
|
+
|
|
247
|
+
const cursor = document.createElement('div');
|
|
248
|
+
cursor.className = 'cb-cursor';
|
|
249
|
+
cursor.innerHTML = '<svg viewBox="0 0 24 24" width="56" height="56" style="display:block"><path d="M5 3 L19 12 L12 13 L15 20 L12 21 L9 14 L5 18 Z" fill="#ffffff" stroke="#000000" stroke-width="1.4" stroke-linejoin="round" stroke-linecap="round"/></svg>';
|
|
250
|
+
document.body.appendChild(cursor);
|
|
251
|
+
|
|
252
|
+
// ─── Smooth cursor animation ───
|
|
253
|
+
// CDP-driven clicks teleport: we get a single mousedown at target coords
|
|
254
|
+
// with no preceding mousemove. To make the cursor's motion feel natural,
|
|
255
|
+
// animate from the last known position to the new one over ~280ms with
|
|
256
|
+
// ease-out. requestAnimationFrame keeps it at display refresh rate.
|
|
257
|
+
//
|
|
258
|
+
// Persist position across navigations via sessionStorage so the cursor
|
|
259
|
+
// resumes exactly where it was on the prior page (e.g. clicked a link in
|
|
260
|
+
// the nav, new page loads — cursor should still be at the link, not jump
|
|
261
|
+
// back to a default spot).
|
|
262
|
+
let curX, curY;
|
|
263
|
+
try {
|
|
264
|
+
const saved = JSON.parse(sessionStorage.getItem('__cbCursor') || 'null');
|
|
265
|
+
if (saved && typeof saved.x === 'number' && typeof saved.y === 'number') {
|
|
266
|
+
curX = Math.min(Math.max(0, saved.x), window.innerWidth);
|
|
267
|
+
curY = Math.min(Math.max(0, saved.y), window.innerHeight);
|
|
268
|
+
}
|
|
269
|
+
} catch {}
|
|
270
|
+
if (typeof curX !== 'number') {
|
|
271
|
+
curX = window.innerWidth / 2;
|
|
272
|
+
curY = 80;
|
|
273
|
+
}
|
|
274
|
+
cursor.style.transform = 'translate(' + (curX - 5) + 'px, ' + (curY - 4) + 'px)';
|
|
275
|
+
const persistCursor = () => {
|
|
276
|
+
try { sessionStorage.setItem('__cbCursor', JSON.stringify({ x: curX, y: curY })); } catch {}
|
|
277
|
+
};
|
|
278
|
+
let animRaf = 0;
|
|
279
|
+
const easeOutCubic = (t) => 1 - Math.pow(1 - t, 3);
|
|
280
|
+
const setCursor = (x, y) => {
|
|
281
|
+
curX = x; curY = y;
|
|
282
|
+
// Offset so the SVG's hotspot (top-left of the arrow tip) sits at the
|
|
283
|
+
// actual click coordinates. With a 56px sprite we offset by ~8/6.
|
|
284
|
+
cursor.style.transform = 'translate3d(' + (x - 8) + 'px, ' + (y - 6) + 'px, 0)';
|
|
285
|
+
persistCursor();
|
|
286
|
+
};
|
|
287
|
+
// Tighter ease curve — snappier acceleration, gentler arrival. Distance-
|
|
288
|
+
// proportional duration so the cursor never crawls on long hops or jitters
|
|
289
|
+
// on tiny ones.
|
|
290
|
+
const easeOutQuart = (t) => 1 - Math.pow(1 - t, 4);
|
|
291
|
+
const moveCursorTo = (x, y, durationMs) => {
|
|
292
|
+
if (animRaf) cancelAnimationFrame(animRaf);
|
|
293
|
+
const startX = curX, startY = curY;
|
|
294
|
+
const dx = x - startX, dy = y - startY;
|
|
295
|
+
const dist = Math.hypot(dx, dy);
|
|
296
|
+
// 200ms minimum for visibility; 600ms cap for very long hops; ~3px/ms
|
|
297
|
+
// travel speed in between feels like a deliberate human reach.
|
|
298
|
+
const dur = Math.min(600, Math.max(200, durationMs ?? (180 + dist * 0.55)));
|
|
299
|
+
const t0 = performance.now();
|
|
300
|
+
const tick = () => {
|
|
301
|
+
const t = Math.min(1, (performance.now() - t0) / dur);
|
|
302
|
+
const e = easeOutQuart(t);
|
|
303
|
+
setCursor(startX + dx * e, startY + dy * e);
|
|
304
|
+
if (t < 1) animRaf = requestAnimationFrame(tick);
|
|
305
|
+
else animRaf = 0;
|
|
306
|
+
};
|
|
307
|
+
animRaf = requestAnimationFrame(tick);
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
const wake = (x, y) => {
|
|
311
|
+
// Two layered shapes: filled radial (the splash) + ring (the wave).
|
|
312
|
+
const w1 = document.createElement('div');
|
|
313
|
+
w1.className = 'cb-wake cb-wake-1';
|
|
314
|
+
w1.style.left = x + 'px'; w1.style.top = y + 'px';
|
|
315
|
+
const w2 = document.createElement('div');
|
|
316
|
+
w2.className = 'cb-wake cb-wake-2';
|
|
317
|
+
w2.style.left = x + 'px'; w2.style.top = y + 'px';
|
|
318
|
+
document.body.appendChild(w1);
|
|
319
|
+
document.body.appendChild(w2);
|
|
320
|
+
setTimeout(() => { w1.remove(); w2.remove(); }, 950);
|
|
321
|
+
};
|
|
322
|
+
|
|
323
|
+
document.addEventListener('mousemove', (e) => moveCursorTo(e.clientX, e.clientY), true);
|
|
324
|
+
document.addEventListener('mousedown', (e) => {
|
|
325
|
+
moveCursorTo(e.clientX, e.clientY, 220);
|
|
326
|
+
// Delay the wake slightly so it appears after the cursor lands.
|
|
327
|
+
setTimeout(() => wake(e.clientX, e.clientY), 180);
|
|
328
|
+
}, true);
|
|
329
|
+
|
|
330
|
+
// ─── Scroll indicator ───
|
|
331
|
+
// Shown briefly when wheel events or programmatic scroll happens. Direction
|
|
332
|
+
// arrow flips based on deltaY sign. Auto-hides after activity stops.
|
|
333
|
+
let scrollIndicator = null;
|
|
334
|
+
let scrollDir = 'down';
|
|
335
|
+
let scrollHideTimer = 0;
|
|
336
|
+
const ensureScrollIndicator = () => {
|
|
337
|
+
if (scrollIndicator) return scrollIndicator;
|
|
338
|
+
scrollIndicator = document.createElement('div');
|
|
339
|
+
scrollIndicator.className = 'cb-scroll-indicator';
|
|
340
|
+
scrollIndicator.innerHTML = '<svg class="cb-scroll-arrow" viewBox="0 0 24 24" width="18" height="18"><path d="M7 10 L12 15 L17 10" stroke="white" stroke-width="2.4" fill="none" stroke-linecap="round" stroke-linejoin="round"/></svg>';
|
|
341
|
+
document.body.appendChild(scrollIndicator);
|
|
342
|
+
return scrollIndicator;
|
|
343
|
+
};
|
|
344
|
+
const showScroll = (dy) => {
|
|
345
|
+
const el = ensureScrollIndicator();
|
|
346
|
+
const dir = dy >= 0 ? 'down' : 'up';
|
|
347
|
+
if (dir !== scrollDir) {
|
|
348
|
+
scrollDir = dir;
|
|
349
|
+
el.classList.toggle('cb-scroll-up', dir === 'up');
|
|
350
|
+
}
|
|
351
|
+
el.classList.add('cb-show');
|
|
352
|
+
if (scrollHideTimer) clearTimeout(scrollHideTimer);
|
|
353
|
+
scrollHideTimer = setTimeout(() => el.classList.remove('cb-show'), 700);
|
|
354
|
+
};
|
|
355
|
+
document.addEventListener('wheel', (e) => showScroll(e.deltaY), { capture: true, passive: true });
|
|
356
|
+
// Programmatic scroll (window.scrollBy / scrollTo) doesn't fire wheel —
|
|
357
|
+
// hook the scroll event too with a direction inferred from delta.
|
|
358
|
+
let lastScrollY = window.scrollY;
|
|
359
|
+
document.addEventListener('scroll', () => {
|
|
360
|
+
const cur = window.scrollY;
|
|
361
|
+
const dy = cur - lastScrollY;
|
|
362
|
+
lastScrollY = cur;
|
|
363
|
+
if (Math.abs(dy) > 1) showScroll(dy);
|
|
364
|
+
}, { capture: true, passive: true });
|
|
365
|
+
|
|
366
|
+
// ─── Action banner ───
|
|
367
|
+
let banner = null;
|
|
368
|
+
let bannerTimer = null;
|
|
369
|
+
window.__cbShowAction = (text) => {
|
|
370
|
+
if (!banner) {
|
|
371
|
+
banner = document.createElement('div');
|
|
372
|
+
banner.className = 'cb-action-banner';
|
|
373
|
+
document.body.appendChild(banner);
|
|
374
|
+
}
|
|
375
|
+
banner.textContent = text;
|
|
376
|
+
banner.classList.add('cb-show');
|
|
377
|
+
if (bannerTimer) clearTimeout(bannerTimer);
|
|
378
|
+
bannerTimer = setTimeout(() => { if (banner) banner.classList.remove('cb-show'); }, 2600);
|
|
379
|
+
};
|
|
380
|
+
|
|
381
|
+
// ─── Manual cursor positioning ───
|
|
382
|
+
// Let the journey runner pre-position the cursor before a click so the
|
|
383
|
+
// animated travel from previous spot to target is visible (clicks otherwise
|
|
384
|
+
// happen as a single mousedown, animation feels like teleport).
|
|
385
|
+
window.__cbMoveCursor = (x, y, dur) => moveCursorTo(x, y, dur);
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
// Defer install until AFTER framework hydration completes. Use
|
|
389
|
+
// requestIdleCallback if available — it fires after main-thread work like
|
|
390
|
+
// hydration settles. Falls back to setTimeout. Modifying the DOM too early
|
|
391
|
+
// races with React/Next.js hydration ("Application error: client-side
|
|
392
|
+
// exception"); too late and the viewer doesn't see the cursor in time.
|
|
393
|
+
const scheduleInstall = () => {
|
|
394
|
+
if (typeof window.requestIdleCallback === 'function') {
|
|
395
|
+
window.requestIdleCallback(install, { timeout: 600 });
|
|
396
|
+
} else {
|
|
397
|
+
setTimeout(install, 100);
|
|
398
|
+
}
|
|
399
|
+
};
|
|
400
|
+
if (document.readyState !== 'loading') {
|
|
401
|
+
scheduleInstall();
|
|
402
|
+
} else {
|
|
403
|
+
document.addEventListener('DOMContentLoaded', scheduleInstall, { once: true });
|
|
404
|
+
}
|
|
405
|
+
})();`;
|
|
406
|
+
/**
|
|
407
|
+
* In-browser attention heatmap. Renders a canvas overlay positioned over the
|
|
408
|
+
* viewport and recomputes saliency directly from the DOM on scroll, resize,
|
|
409
|
+
* and DOM mutation. The heatmap is drawn INSIDE the chromium page, which
|
|
410
|
+
* means VNC captures it natively — no Node-side image generation, no SSE
|
|
411
|
+
* payload, no dashboard-side overlay positioning. Per-frame compute is
|
|
412
|
+
* sub-millisecond on modern pages because we're just reading bounding boxes.
|
|
413
|
+
*
|
|
414
|
+
* Config is injected as `window.__cbAttentionConfig` by the journey runner.
|
|
415
|
+
*/
|
|
416
|
+
const ATTENTION_OVERLAY_SCRIPT = `(() => {
|
|
417
|
+
if (window.__cbAttentionInstalled || window.__cbAttentionPending) return;
|
|
418
|
+
window.__cbAttentionPending = true;
|
|
419
|
+
|
|
420
|
+
const install = () => {
|
|
421
|
+
if (window.__cbAttentionInstalled) return;
|
|
422
|
+
if (!document.documentElement || !document.body) return;
|
|
423
|
+
window.__cbAttentionInstalled = true;
|
|
424
|
+
|
|
425
|
+
const config = window.__cbAttentionConfig || { typeWeights: {}, personaMods: {}, goal: "" };
|
|
426
|
+
const cellSize = 16;
|
|
427
|
+
|
|
428
|
+
const css = \`
|
|
429
|
+
.cb-attn-canvas {
|
|
430
|
+
position: fixed; left: 0; top: 0;
|
|
431
|
+
width: 100vw; height: 100vh;
|
|
432
|
+
pointer-events: none;
|
|
433
|
+
z-index: 2147483640;
|
|
434
|
+
opacity: 0.5;
|
|
435
|
+
/* Moderate blur smears the per-cell rectangles into a smooth gradient.
|
|
436
|
+
Lower than 28px to reduce GPU load — heavy blur on a fullscreen
|
|
437
|
+
canvas with --disable-gpu can cause frame drops / black flashes. */
|
|
438
|
+
filter: blur(18px);
|
|
439
|
+
/* Promote canvas to its own compositor layer so blur computes once
|
|
440
|
+
per repaint, not on every parent reflow. */
|
|
441
|
+
transform: translateZ(0);
|
|
442
|
+
backface-visibility: hidden;
|
|
443
|
+
}
|
|
444
|
+
\`;
|
|
445
|
+
const style = document.createElement('style');
|
|
446
|
+
style.textContent = css;
|
|
447
|
+
(document.head || document.documentElement).appendChild(style);
|
|
448
|
+
|
|
449
|
+
const canvas = document.createElement('canvas');
|
|
450
|
+
canvas.className = 'cb-attn-canvas';
|
|
451
|
+
document.body.appendChild(canvas);
|
|
452
|
+
const ctx = canvas.getContext('2d');
|
|
453
|
+
|
|
454
|
+
// Color stops — blue (low) → green → yellow → red (high). Alpha grows
|
|
455
|
+
// with saliency so cold areas are nearly transparent.
|
|
456
|
+
const stops = [
|
|
457
|
+
{ t: 0.00, r: 0, g: 0, b: 128, a: 0.0 },
|
|
458
|
+
{ t: 0.20, r: 0, g: 0, b: 255, a: 0.35 },
|
|
459
|
+
{ t: 0.40, r: 0, g: 255, b: 255, a: 0.5 },
|
|
460
|
+
{ t: 0.55, r: 0, g: 255, b: 0, a: 0.6 },
|
|
461
|
+
{ t: 0.70, r: 255, g: 255, b: 0, a: 0.7 },
|
|
462
|
+
{ t: 0.85, r: 255, g: 128, b: 0, a: 0.8 },
|
|
463
|
+
{ t: 1.00, r: 255, g: 0, b: 0, a: 0.9 },
|
|
464
|
+
];
|
|
465
|
+
const sample = (v) => {
|
|
466
|
+
v = Math.max(0, Math.min(1, v));
|
|
467
|
+
let lo = 0;
|
|
468
|
+
while (lo < stops.length - 1 && stops[lo + 1].t < v) lo++;
|
|
469
|
+
const hi = Math.min(lo + 1, stops.length - 1);
|
|
470
|
+
const a = stops[lo], b = stops[hi];
|
|
471
|
+
const span = b.t - a.t || 1;
|
|
472
|
+
const k = (v - a.t) / span;
|
|
473
|
+
return [
|
|
474
|
+
Math.round(a.r + (b.r - a.r) * k),
|
|
475
|
+
Math.round(a.g + (b.g - a.g) * k),
|
|
476
|
+
Math.round(a.b + (b.b - a.b) * k),
|
|
477
|
+
a.a + (b.a - a.a) * k,
|
|
478
|
+
];
|
|
479
|
+
};
|
|
480
|
+
|
|
481
|
+
// Classify a DOM element into one of the persona's weighted categories.
|
|
482
|
+
const classify = (el) => {
|
|
483
|
+
const tag = el.tagName.toLowerCase();
|
|
484
|
+
const role = el.getAttribute('role') || '';
|
|
485
|
+
if (el.closest('nav, [role="navigation"]')) return 'navigation';
|
|
486
|
+
if (tag === 'button' || role === 'button' || role === 'link' && tag !== 'a') return 'cta';
|
|
487
|
+
if (tag === 'a') {
|
|
488
|
+
// Prominent CTAs: large buttons styled as anchors
|
|
489
|
+
const txt = (el.innerText || '').trim();
|
|
490
|
+
if (txt && txt.length < 40 && /sign\\s*up|get\\s*started|try|buy|start|create|register|apply/i.test(txt)) return 'cta';
|
|
491
|
+
return 'link';
|
|
492
|
+
}
|
|
493
|
+
if (/^h[1-6]$/.test(tag) || role === 'heading') return 'heading';
|
|
494
|
+
if (tag === 'input' || tag === 'select' || tag === 'textarea') {
|
|
495
|
+
if (/^search$/i.test(el.getAttribute('type') || '') || /search/i.test(el.getAttribute('placeholder') || '')) return 'search';
|
|
496
|
+
return 'form';
|
|
497
|
+
}
|
|
498
|
+
if (tag === 'img' || tag === 'svg') {
|
|
499
|
+
if (el.getAttribute('aria-hidden') === 'true' || el.getAttribute('alt') === '') return 'decorative';
|
|
500
|
+
return 'image';
|
|
501
|
+
}
|
|
502
|
+
if (/\\$\\d|\\d+\\s*(usd|eur|gbp|\\$|€|£)/i.test(el.textContent || '')) return 'price';
|
|
503
|
+
if (el.getAttribute('role') === 'alert' || /\\berror\\b/i.test(el.className || '')) return 'error';
|
|
504
|
+
return 'content';
|
|
505
|
+
};
|
|
506
|
+
|
|
507
|
+
// Sum of weight from goal-keyword matches in element text.
|
|
508
|
+
const goalWords = (config.goal || '').toLowerCase()
|
|
509
|
+
.split(/\\s+/).filter(w => w.length > 2 && !/^(the|a|an|to|for|of|in|on|at|is|it|my|i|me|and|or|how|do|can)$/.test(w));
|
|
510
|
+
const goalRelevance = (text) => {
|
|
511
|
+
if (!text || !goalWords.length) return 0;
|
|
512
|
+
const t = text.toLowerCase();
|
|
513
|
+
let m = 0;
|
|
514
|
+
for (const w of goalWords) if (t.includes(w)) m++;
|
|
515
|
+
return (m / goalWords.length) * 2.0;
|
|
516
|
+
};
|
|
517
|
+
|
|
518
|
+
let lastCells = null;
|
|
519
|
+
let lastDims = { cols: 0, rows: 0, w: 0, h: 0 };
|
|
520
|
+
let frameId = 0;
|
|
521
|
+
const recompute = () => {
|
|
522
|
+
const w = window.innerWidth, h = window.innerHeight;
|
|
523
|
+
const dpr = window.devicePixelRatio || 1;
|
|
524
|
+
if (canvas.width !== w * dpr || canvas.height !== h * dpr) {
|
|
525
|
+
canvas.width = w * dpr;
|
|
526
|
+
canvas.height = h * dpr;
|
|
527
|
+
ctx.scale(dpr, dpr);
|
|
528
|
+
}
|
|
529
|
+
const cols = Math.ceil(w / cellSize);
|
|
530
|
+
const rows = Math.ceil(h / cellSize);
|
|
531
|
+
const cells = new Float32Array(cols * rows);
|
|
532
|
+
|
|
533
|
+
// Sample interactive + structural elements; cap to 600 to bound work
|
|
534
|
+
const sel = 'button, a, [role="button"], [role="link"], [role="heading"], h1, h2, h3, h4, input, select, textarea, [data-cb-cta], img, label, p, li';
|
|
535
|
+
const els = document.querySelectorAll(sel);
|
|
536
|
+
const limit = Math.min(els.length, 600);
|
|
537
|
+
for (let i = 0; i < limit; i++) {
|
|
538
|
+
const el = els[i];
|
|
539
|
+
const r = el.getBoundingClientRect();
|
|
540
|
+
if (r.bottom < 0 || r.top > h || r.right < 0 || r.left > w) continue;
|
|
541
|
+
if (r.width < 4 || r.height < 4) continue;
|
|
542
|
+
const cs = window.getComputedStyle(el);
|
|
543
|
+
if (cs.display === 'none' || cs.visibility === 'hidden' || cs.opacity === '0') continue;
|
|
544
|
+
|
|
545
|
+
const type = classify(el);
|
|
546
|
+
let weight = (config.typeWeights && config.typeWeights[type]) || 0.2;
|
|
547
|
+
const mod = (config.personaMods && config.personaMods[type]) || 1.0;
|
|
548
|
+
weight *= mod;
|
|
549
|
+
const text = (el.innerText || el.textContent || '').slice(0, 200);
|
|
550
|
+
const gr = goalRelevance(text);
|
|
551
|
+
if (gr > 0) weight *= (1 + gr);
|
|
552
|
+
|
|
553
|
+
const x1 = Math.max(0, Math.floor(r.left / cellSize));
|
|
554
|
+
const x2 = Math.min(cols - 1, Math.floor(r.right / cellSize));
|
|
555
|
+
const y1 = Math.max(0, Math.floor(r.top / cellSize));
|
|
556
|
+
const y2 = Math.min(rows - 1, Math.floor(r.bottom / cellSize));
|
|
557
|
+
for (let rr = y1; rr <= y2; rr++) {
|
|
558
|
+
for (let cc = x1; cc <= x2; cc++) {
|
|
559
|
+
const idx = rr * cols + cc;
|
|
560
|
+
if (cells[idx] < weight) cells[idx] = weight;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Normalize
|
|
566
|
+
let max = 0;
|
|
567
|
+
for (let i = 0; i < cells.length; i++) if (cells[i] > max) max = cells[i];
|
|
568
|
+
if (max === 0) max = 1;
|
|
569
|
+
|
|
570
|
+
// Render — clearRect first, then per-cell fill. The CSS filter blur
|
|
571
|
+
// (28px) smears these rectangles into a smooth gradient, so we keep the
|
|
572
|
+
// threshold low enough that mid-saliency cells contribute their colors
|
|
573
|
+
// to the blur halo rather than leaving cold gaps.
|
|
574
|
+
ctx.clearRect(0, 0, w, h);
|
|
575
|
+
for (let r0 = 0; r0 < rows; r0++) {
|
|
576
|
+
for (let c0 = 0; c0 < cols; c0++) {
|
|
577
|
+
const v = cells[r0 * cols + c0] / max;
|
|
578
|
+
if (v < 0.03) continue;
|
|
579
|
+
const [rr, gg, bb, aa] = sample(v);
|
|
580
|
+
ctx.fillStyle = 'rgba(' + rr + ',' + gg + ',' + bb + ',' + aa + ')';
|
|
581
|
+
ctx.fillRect(c0 * cellSize, r0 * cellSize, cellSize, cellSize);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
lastCells = cells;
|
|
585
|
+
lastDims = { cols, rows, w, h };
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
const schedule = () => {
|
|
589
|
+
if (frameId) cancelAnimationFrame(frameId);
|
|
590
|
+
frameId = requestAnimationFrame(() => { frameId = 0; recompute(); });
|
|
591
|
+
};
|
|
592
|
+
|
|
593
|
+
window.addEventListener('scroll', schedule, { capture: true, passive: true });
|
|
594
|
+
window.addEventListener('resize', schedule);
|
|
595
|
+
document.addEventListener('scroll', schedule, { capture: true, passive: true });
|
|
596
|
+
|
|
597
|
+
// Watch for major DOM changes — debounce to avoid mutation storms
|
|
598
|
+
let mutTimer = 0;
|
|
599
|
+
new MutationObserver(() => {
|
|
600
|
+
if (mutTimer) clearTimeout(mutTimer);
|
|
601
|
+
mutTimer = setTimeout(schedule, 300);
|
|
602
|
+
}).observe(document.body, { childList: true, subtree: true, attributes: false });
|
|
603
|
+
|
|
604
|
+
schedule();
|
|
605
|
+
// Initial settle re-render after fonts/images load
|
|
606
|
+
setTimeout(schedule, 600);
|
|
607
|
+
setTimeout(schedule, 1500);
|
|
608
|
+
|
|
609
|
+
// Public API
|
|
610
|
+
window.__cbAttentionToggle = (visible) => {
|
|
611
|
+
canvas.style.display = visible === false ? 'none' : '';
|
|
612
|
+
};
|
|
613
|
+
window.__cbAttentionRefresh = schedule;
|
|
614
|
+
};
|
|
615
|
+
|
|
616
|
+
if (typeof window.requestIdleCallback === 'function') {
|
|
617
|
+
window.requestIdleCallback(install, { timeout: 800 });
|
|
618
|
+
} else {
|
|
619
|
+
setTimeout(install, 150);
|
|
620
|
+
}
|
|
621
|
+
})();`;
|
|
622
|
+
/**
|
|
623
|
+
* Build the JSON config bundle that drives the in-browser attention heatmap.
|
|
624
|
+
* The browser-side script doesn't have access to the SDK's persona profiles,
|
|
625
|
+
* so we precompute the type-weight × persona-modifier table here and ship it.
|
|
626
|
+
*/
|
|
627
|
+
function buildAttentionConfig(personaName, goal, traits) {
|
|
628
|
+
// Mirror of ELEMENT_TYPE_WEIGHTS in attention-transport.ts. Kept in sync by
|
|
629
|
+
// hand — small enough that drift hasn't been a problem.
|
|
630
|
+
const typeWeights = {
|
|
631
|
+
cta: 1.0, heading: 0.8, form: 0.7, search: 0.7,
|
|
632
|
+
image: 0.5, navigation: 0.4, link: 0.3,
|
|
633
|
+
content: 0.2, decorative: 0.1, price: 0.9, error: 0.9,
|
|
634
|
+
};
|
|
635
|
+
// Trait-driven modifiers. Mirrors computeTraitModifiers in attention-transport.
|
|
636
|
+
const t = (k) => traits?.[k] ?? 0.5;
|
|
637
|
+
const personaMods = {
|
|
638
|
+
cta: 1.0 + (1.0 - t("patience")) * 0.5,
|
|
639
|
+
search: 0.7 + (1.0 - t("patience")) * 0.6,
|
|
640
|
+
content: t("readingTendency") > 0.6 ? 0.4 + t("readingTendency") * 0.4 : 0.2 + t("patience") * 0.4,
|
|
641
|
+
heading: 0.8 + t("curiosity") * 0.4,
|
|
642
|
+
image: 0.5 + t("curiosity") * 0.5,
|
|
643
|
+
navigation: 0.4 + (1.0 - t("siteFamiliarity")) * 0.6,
|
|
644
|
+
error: 0.5 + (1.0 - t("selfEfficacy")) * 0.5,
|
|
645
|
+
price: 0.5 + (1.0 - t("riskTolerance")) * 0.5,
|
|
646
|
+
decorative: 0.1 + (1.0 - t("metacognitivePlanning")) * 0.4,
|
|
647
|
+
form: 0.5 + t("proceduralFluency") * 0.3,
|
|
648
|
+
link: 1.0,
|
|
649
|
+
};
|
|
650
|
+
return { typeWeights, personaMods, goal: goal || "", persona: personaName };
|
|
651
|
+
}
|
|
652
|
+
// Note: CognitiveJourneyOptions accepts `device` to emulate a specific device
|
|
653
|
+
// preset (mobile/tablet/desktop or "iPhone 15", etc.). Wired in below alongside
|
|
654
|
+
// the other browserConfig overrides.
|
|
116
655
|
/**
|
|
117
656
|
* Run an autonomous cognitive journey using the Claude API.
|
|
118
657
|
*
|
|
@@ -189,6 +728,22 @@ export async function runCognitiveJourney(options) {
|
|
|
189
728
|
}
|
|
190
729
|
const profile = getCognitiveProfile(personaObj);
|
|
191
730
|
const traits = profile.traits;
|
|
731
|
+
// Build the InputTraits subset used by human-input.ts. CognitiveTraits has
|
|
732
|
+
// proceduralFluency / patience / readingTendency directly; motorPrecision
|
|
733
|
+
// doesn't exist as a cognitive trait, so we proxy it from proceduralFluency
|
|
734
|
+
// (motor and procedural skill correlate strongly: r ≈ 0.5-0.7 in the
|
|
735
|
+
// psychomotor literature). Personas with explicit motor disability traits
|
|
736
|
+
// (low-vision-magnified, motor-impairment-tremor) get extra penalty so
|
|
737
|
+
// their clicks visibly miss more often.
|
|
738
|
+
const aTraits = personaObj.accessibilityTraits;
|
|
739
|
+
const motorBase = traits.proceduralFluency ?? 0.5;
|
|
740
|
+
const motorPenalty = aTraits?.motorControl !== undefined ? aTraits.motorControl : 1.0;
|
|
741
|
+
const inputTraits = {
|
|
742
|
+
proceduralFluency: traits.proceduralFluency ?? 0.5,
|
|
743
|
+
motorPrecision: Math.max(0.1, Math.min(1, motorBase * motorPenalty)),
|
|
744
|
+
patience: traits.patience ?? 0.5,
|
|
745
|
+
readingTendency: traits.readingTendency ?? 0.5,
|
|
746
|
+
};
|
|
192
747
|
// Calculate abandonment thresholds
|
|
193
748
|
const thresholds = {
|
|
194
749
|
patienceMin: 0.1,
|
|
@@ -294,7 +849,7 @@ export async function runCognitiveJourney(options) {
|
|
|
294
849
|
// Create browser with timezone/locale/geolocation if specified
|
|
295
850
|
const browserConfig = {
|
|
296
851
|
headless,
|
|
297
|
-
persistent: true,
|
|
852
|
+
persistent: options.persistent ?? true,
|
|
298
853
|
};
|
|
299
854
|
if (effectiveLocation.timezone)
|
|
300
855
|
browserConfig.timezone = effectiveLocation.timezone;
|
|
@@ -302,8 +857,93 @@ export async function runCognitiveJourney(options) {
|
|
|
302
857
|
browserConfig.locale = effectiveLocation.locale;
|
|
303
858
|
if (effectiveLocation.geolocation)
|
|
304
859
|
browserConfig.geolocation = effectiveLocation.geolocation;
|
|
860
|
+
if (options.launchEnv)
|
|
861
|
+
browserConfig.launchEnv = options.launchEnv;
|
|
862
|
+
if (options.device)
|
|
863
|
+
browserConfig.device = options.device;
|
|
864
|
+
// Live viewer alignment: chromium runs in a 1920×1080 OS window on Xvfb;
|
|
865
|
+
// browser chrome (tabs + URL bar) takes ~85px from the top, so the visible
|
|
866
|
+
// page area is 1920×995. We size the Playwright viewport to MATCH that
|
|
867
|
+
// visible area — that way the attention heatmap (computed for window.inner*)
|
|
868
|
+
// is exactly the right shape to overlay on the VNC stream.
|
|
869
|
+
// Window size stays at 1920×1080 so chromium fills Xvfb edge-to-edge.
|
|
870
|
+
// Dashboard overlay positions: top:7.87% (85/1080), height:92.13% (995/1080).
|
|
871
|
+
if (options.enableMouseOverlay) {
|
|
872
|
+
browserConfig.viewportWidth = 1920;
|
|
873
|
+
browserConfig.viewportHeight = 995;
|
|
874
|
+
browserConfig.windowWidth = 1920;
|
|
875
|
+
browserConfig.windowHeight = 1080;
|
|
876
|
+
// Heavy university / news / e-commerce sites often need >30s to reach
|
|
877
|
+
// domcontentloaded with all their analytics/CDN scripts. Live journeys
|
|
878
|
+
// get a 60s navigation budget per attempt.
|
|
879
|
+
browserConfig.timeout = 60000;
|
|
880
|
+
// When a click opens target="_blank" or window.open, follow the new tab
|
|
881
|
+
// so VNC keeps showing where the persona actually went.
|
|
882
|
+
browserConfig.followPopups = true;
|
|
883
|
+
}
|
|
305
884
|
const browser = new CBrowser(browserConfig);
|
|
306
|
-
|
|
885
|
+
// Helper that re-injects the mouse overlay. The overlay script is idempotent
|
|
886
|
+
// (gates on window.__cbHelperInstalled), so calling this after every navigation
|
|
887
|
+
// is safe and ensures the overlay always survives full page loads.
|
|
888
|
+
const installMouseOverlay = async () => {
|
|
889
|
+
if (!options.enableMouseOverlay)
|
|
890
|
+
return;
|
|
891
|
+
try {
|
|
892
|
+
const page = await browser.getPage();
|
|
893
|
+
await page.evaluate(MOUSE_OVERLAY_SCRIPT);
|
|
894
|
+
}
|
|
895
|
+
catch (e) {
|
|
896
|
+
console.warn(`[cognitive] Mouse overlay inject failed: ${e.message}`);
|
|
897
|
+
}
|
|
898
|
+
};
|
|
899
|
+
// Add the script as a context-level init script too — this catches new
|
|
900
|
+
// pages opened by JS (target=_blank, window.open) without us having to
|
|
901
|
+
// re-inject in the journey loop.
|
|
902
|
+
if (options.enableMouseOverlay) {
|
|
903
|
+
try {
|
|
904
|
+
const initialPage = await browser.getPage();
|
|
905
|
+
await initialPage.context().addInitScript(MOUSE_OVERLAY_SCRIPT);
|
|
906
|
+
// In-browser attention heatmap. Renders to a canvas inside the page so
|
|
907
|
+
// VNC captures it natively and updates instantly with scroll/resize.
|
|
908
|
+
// Persona modifiers (type weights × cognitive traits) are precomputed
|
|
909
|
+
// here in Node and serialized into the page as window.__cbAttentionConfig.
|
|
910
|
+
const attentionConfig = buildAttentionConfig(personaObj.name, options.goal, traits);
|
|
911
|
+
const configBootstrap = `window.__cbAttentionConfig = ${JSON.stringify(attentionConfig)};`;
|
|
912
|
+
await initialPage.context().addInitScript(configBootstrap + ATTENTION_OVERLAY_SCRIPT);
|
|
913
|
+
}
|
|
914
|
+
catch (e) {
|
|
915
|
+
console.warn(`[cognitive] Overlay context init failed: ${e.message}`);
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
// Navigate with retry for transient network errors. Use domcontentloaded
|
|
919
|
+
// strategy (not the default "auto" which tries networkidle first) — heavy
|
|
920
|
+
// sites like .edu / news / e-commerce often never reach networkidle and
|
|
921
|
+
// would burn the full 30s timeout before falling back. Once DOM is
|
|
922
|
+
// interactive the persona can start perceiving; trailing tracker requests
|
|
923
|
+
// shouldn't block. ERR_NETWORK_CHANGED retried up to 4 times.
|
|
924
|
+
// Hard timeout per attempt is browserConfig.timeout (set to 60s above for
|
|
925
|
+
// live mode).
|
|
926
|
+
let lastErr;
|
|
927
|
+
for (let attempt = 1; attempt <= 4; attempt++) {
|
|
928
|
+
try {
|
|
929
|
+
await browser.navigate(options.startUrl, { waitStrategy: "domcontentloaded" });
|
|
930
|
+
lastErr = undefined;
|
|
931
|
+
break;
|
|
932
|
+
}
|
|
933
|
+
catch (navErr) {
|
|
934
|
+
lastErr = navErr;
|
|
935
|
+
const isTransient = /ERR_NETWORK_CHANGED|ERR_CONNECTION|ERR_TIMED_OUT|Timeout/i.test(lastErr.message);
|
|
936
|
+
if (attempt === 4 || !isTransient) {
|
|
937
|
+
if (!isTransient)
|
|
938
|
+
throw lastErr;
|
|
939
|
+
}
|
|
940
|
+
await new Promise(r => setTimeout(r, 2500 * attempt));
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
if (lastErr)
|
|
944
|
+
throw lastErr;
|
|
945
|
+
// Inject into the freshly-loaded page (init script may have raced the navigation)
|
|
946
|
+
await installMouseOverlay();
|
|
307
947
|
// Apply geolocation at runtime as well (ensures permission is granted after context exists)
|
|
308
948
|
if (effectiveLocation.geolocation) {
|
|
309
949
|
await browser.setGeolocationRuntime(effectiveLocation.geolocation);
|
|
@@ -312,8 +952,104 @@ export async function runCognitiveJourney(options) {
|
|
|
312
952
|
const frictionPoints = [];
|
|
313
953
|
const startTime = Date.now();
|
|
314
954
|
const maxSteps = options.maxSteps || 50;
|
|
315
|
-
//
|
|
316
|
-
|
|
955
|
+
// ── Returning-visit familiarity boost ─────────────────────────────────
|
|
956
|
+
// The persona's existing siteFamiliarity trait already drives perception
|
|
957
|
+
// (navigation weighting in attention-transport, scan-path defaults, etc.).
|
|
958
|
+
// Rather than bolt a separate "memory" mechanism on top, we LIFT
|
|
959
|
+
// siteFamiliarity based on prior-visit count so the existing pipeline
|
|
960
|
+
// automatically reflects "this site feels familiar to me." Saturating
|
|
961
|
+
// curve: 0 visits = persona's baseline, 1 visit ≈ +0.22, 3 ≈ +0.53,
|
|
962
|
+
// 5 ≈ +0.72, 10+ ≈ +0.92 — asymptotes near 1.
|
|
963
|
+
if (options.priorMemory && options.priorMemory.visitCount > 0) {
|
|
964
|
+
const v = options.priorMemory.visitCount;
|
|
965
|
+
const familiarityBoost = 1 - Math.exp(-v / 4);
|
|
966
|
+
const baseline = traits.siteFamiliarity ?? 0.5;
|
|
967
|
+
// Boost toward 1.0 from baseline. The persona keeps any fundamental
|
|
968
|
+
// cognitive trait (e.g. low transferLearning still matters) but the
|
|
969
|
+
// domain-specific familiarity component lifts.
|
|
970
|
+
traits.siteFamiliarity = Math.min(1, baseline + (1 - baseline) * familiarityBoost);
|
|
971
|
+
if (options.verbose) {
|
|
972
|
+
console.log(`👁️ siteFamiliarity ${baseline.toFixed(2)} → ${traits.siteFamiliarity.toFixed(2)} (visit #${v + 1})`);
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
// ── Time-of-day / circadian modulation ────────────────────────────────
|
|
976
|
+
// Layered ON TOP of the persona's base traits. Same persona at 11pm Fri
|
|
977
|
+
// is less patient, less curious about exploring, and more distractible
|
|
978
|
+
// than at 10am Tue. Modulators are additive deltas clamped to [-0.4, 0.4].
|
|
979
|
+
// Falls back to server-current time + UTC if no temporalContext provided.
|
|
980
|
+
let temporalModulators = null;
|
|
981
|
+
let temporalState = null;
|
|
982
|
+
try {
|
|
983
|
+
const { getTemporalState, computeTemporalModulators } = await import("./temporal.js");
|
|
984
|
+
const tCtx = options.temporalContext ?? { now: new Date() };
|
|
985
|
+
temporalState = getTemporalState(tCtx);
|
|
986
|
+
temporalModulators = computeTemporalModulators(temporalState);
|
|
987
|
+
// Apply trait deltas
|
|
988
|
+
traits.patience = Math.max(0, Math.min(1, (traits.patience ?? 0.5) + temporalModulators.patienceDelta));
|
|
989
|
+
if (traits.workingMemory !== undefined) {
|
|
990
|
+
traits.workingMemory = Math.max(0, Math.min(1, traits.workingMemory + temporalModulators.workingMemoryDelta));
|
|
991
|
+
}
|
|
992
|
+
if (traits.curiosity !== undefined) {
|
|
993
|
+
traits.curiosity = Math.max(0, Math.min(1, traits.curiosity + temporalModulators.curiosityDelta));
|
|
994
|
+
}
|
|
995
|
+
if (traits.metacognitivePlanning !== undefined) {
|
|
996
|
+
traits.metacognitivePlanning = Math.max(0, Math.min(1, traits.metacognitivePlanning + temporalModulators.metacognitiveDelta));
|
|
997
|
+
}
|
|
998
|
+
if (options.verbose) {
|
|
999
|
+
console.log(`🕐 Temporal: ${temporalState.description} (energy ${temporalState.energy.toFixed(2)})`);
|
|
1000
|
+
console.log(` Δ patience ${temporalModulators.patienceDelta >= 0 ? "+" : ""}${temporalModulators.patienceDelta.toFixed(2)}, distraction rate ${(temporalModulators.distractionRate * 100).toFixed(0)}%`);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
catch (e) {
|
|
1004
|
+
console.warn(`[cognitive] temporal modulation skipped: ${e.message}`);
|
|
1005
|
+
}
|
|
1006
|
+
// Build system prompt AFTER the familiarity AND temporal adjustments so
|
|
1007
|
+
// any prompt strings derived from traits use the modulated values. Then
|
|
1008
|
+
// append a returning-visit recall block with concrete observations.
|
|
1009
|
+
let systemPrompt = buildCognitiveSystemPrompt(personaObj, profile, options.goal, thresholds);
|
|
1010
|
+
if (options.priorMemory && options.priorMemory.visitCount > 0) {
|
|
1011
|
+
const m = options.priorMemory;
|
|
1012
|
+
const sinceDays = m.lastVisitAt
|
|
1013
|
+
? Math.round((Date.now() - new Date(m.lastVisitAt.replace(" ", "T") + "Z").getTime()) / (1000 * 60 * 60 * 24))
|
|
1014
|
+
: null;
|
|
1015
|
+
const sinceStr = sinceDays === null
|
|
1016
|
+
? "before"
|
|
1017
|
+
: sinceDays === 0 ? "earlier today" : sinceDays === 1 ? "yesterday" : `${sinceDays} days ago`;
|
|
1018
|
+
const outcomeStr = m.lastOutcome
|
|
1019
|
+
? (m.lastOutcome.achieved
|
|
1020
|
+
? `Last visit you achieved your goal (${m.lastOutcome.reason}).`
|
|
1021
|
+
: `Last visit you abandoned (${m.lastOutcome.reason}).`)
|
|
1022
|
+
: "";
|
|
1023
|
+
const obsStr = m.observations.length
|
|
1024
|
+
? `\nThings you remember about this site:\n${m.observations.map(o => ` • ${o}`).join("\n")}`
|
|
1025
|
+
: "";
|
|
1026
|
+
const baselineStr = m.emotionalBaseline
|
|
1027
|
+
? `\nYour residual feeling about this site (decayed since ${sinceStr}): satisfaction=${m.emotionalBaseline.satisfaction.toFixed(2)}, frustration=${m.emotionalBaseline.frustration.toFixed(2)}.`
|
|
1028
|
+
: "";
|
|
1029
|
+
const memBlock = `
|
|
1030
|
+
|
|
1031
|
+
— RETURNING-VISIT CONTEXT —
|
|
1032
|
+
This is your ${m.visitCount + 1}${ordinalSuffix(m.visitCount + 1)} visit to this site (last visited ${sinceStr}). ${outcomeStr}${obsStr}${baselineStr}
|
|
1033
|
+
Use this memory naturally — don't recite it, but let it shape your patience, your scan path, and your defaults. If your last visit ended in frustration, that residue carries; if you've used the search before, you'll head there again before scrolling.`;
|
|
1034
|
+
systemPrompt += memBlock;
|
|
1035
|
+
}
|
|
1036
|
+
// Append time-of-day context — short, naturalistic. The trait deltas
|
|
1037
|
+
// already shifted the persona's behavior implicitly; this string makes
|
|
1038
|
+
// the framing recallable in monologue ("It's late and I just want this
|
|
1039
|
+
// done").
|
|
1040
|
+
if (temporalState) {
|
|
1041
|
+
systemPrompt += `
|
|
1042
|
+
|
|
1043
|
+
— WHEN —
|
|
1044
|
+
Right now it's ${temporalState.description}.`;
|
|
1045
|
+
}
|
|
1046
|
+
// Seed emotional state from carried-over baseline so the persona starts the
|
|
1047
|
+
// session feeling whatever decayed-residue from last time, not pure neutral.
|
|
1048
|
+
if (options.priorMemory?.emotionalBaseline && state.emotionalState) {
|
|
1049
|
+
const b = options.priorMemory.emotionalBaseline;
|
|
1050
|
+
state.emotionalState.satisfaction = Math.min(1, Math.max(0, state.emotionalState.satisfaction + b.satisfaction));
|
|
1051
|
+
state.emotionalState.frustration = Math.min(1, Math.max(0, state.emotionalState.frustration + b.frustration));
|
|
1052
|
+
}
|
|
317
1053
|
// Conversation history for Claude
|
|
318
1054
|
const messages = [];
|
|
319
1055
|
let goalAchieved = false;
|
|
@@ -322,6 +1058,16 @@ export async function runCognitiveJourney(options) {
|
|
|
322
1058
|
let currentUrl = options.startUrl;
|
|
323
1059
|
let goalEvidence;
|
|
324
1060
|
let failureReason;
|
|
1061
|
+
// Last action outcome — fed into the next step's prompt so the AI gets a
|
|
1062
|
+
// structured "what changed" report instead of having to infer from raw state.
|
|
1063
|
+
// Without this, Claude often reasons as if the page is unchanged when in fact
|
|
1064
|
+
// a navigation happened (or vice versa).
|
|
1065
|
+
let lastActionOutcome = null;
|
|
1066
|
+
let prevClickableTexts = new Set();
|
|
1067
|
+
let prevScrollY = 0;
|
|
1068
|
+
// Per-URL last-captured scroll Y. Drives screenshot gating: capture on first
|
|
1069
|
+
// visit to a URL or when subsequent scroll exceeds 100px since last capture.
|
|
1070
|
+
const lastScreenshotScroll = new Map();
|
|
325
1071
|
// v18.29.0: Step-by-step journey log with path, elements, evidence
|
|
326
1072
|
const journeyLog = [];
|
|
327
1073
|
// Main cognitive loop
|
|
@@ -334,30 +1080,134 @@ export async function runCognitiveJourney(options) {
|
|
|
334
1080
|
abandonmentMessage = `I've spent too long on this (${Math.round(state.timeElapsed)}s). Giving up.`;
|
|
335
1081
|
break;
|
|
336
1082
|
}
|
|
337
|
-
//
|
|
338
|
-
|
|
339
|
-
//
|
|
1083
|
+
// Ambient distraction roll — phone buzz, kid asks question, doorbell.
|
|
1084
|
+
// Rate driven by circadian energy: ~3% per step at peak alertness, up
|
|
1085
|
+
// to ~25% when tired. Skipped on step 1 (just landed, focused).
|
|
1086
|
+
if (temporalModulators && step > 1) {
|
|
1087
|
+
const { rollDistraction } = await import("./temporal.js");
|
|
1088
|
+
const distractMs = rollDistraction(temporalModulators);
|
|
1089
|
+
if (distractMs > 0) {
|
|
1090
|
+
if (options.verbose)
|
|
1091
|
+
console.log(`📵 Distraction: ${(distractMs / 1000).toFixed(1)}s pause`);
|
|
1092
|
+
await new Promise(r => setTimeout(r, distractMs));
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
// Get page info first (cheap) so we can decide whether to screenshot.
|
|
1096
|
+
// bringToFront ensures the persona's tab is the visible one in chromium
|
|
1097
|
+
// — important for VNC viewers when a click opened a new tab.
|
|
340
1098
|
const page = await browser.getPage();
|
|
1099
|
+
try {
|
|
1100
|
+
await page.bringToFront();
|
|
1101
|
+
}
|
|
1102
|
+
catch { /* page may have been closed */ }
|
|
1103
|
+
// Browser health check — chromium's renderer can freeze on heavy pages,
|
|
1104
|
+
// leaving the VNC stream black while the worker keeps stepping. A trivial
|
|
1105
|
+
// page.evaluate that times out >5s means the renderer is unresponsive.
|
|
1106
|
+
// Abort the journey rather than wedging until the 10-min hard cap.
|
|
1107
|
+
try {
|
|
1108
|
+
await Promise.race([
|
|
1109
|
+
page.evaluate("1+1"),
|
|
1110
|
+
new Promise((_, rej) => setTimeout(() => rej(new Error("renderer health check timed out")), 5000)),
|
|
1111
|
+
]);
|
|
1112
|
+
}
|
|
1113
|
+
catch (healthErr) {
|
|
1114
|
+
console.warn(`[cognitive] step ${step}: ${healthErr.message} — aborting journey`);
|
|
1115
|
+
abandonmentReason = "timeout";
|
|
1116
|
+
abandonmentMessage = `Browser became unresponsive on step ${step}. (${healthErr.message})`;
|
|
1117
|
+
break;
|
|
1118
|
+
}
|
|
341
1119
|
const pageTitle = await page.title() || "Current Page";
|
|
342
1120
|
const availableElements = await browser.getAvailableClickables();
|
|
343
1121
|
const availableInputs = await browser.getAvailableInputs();
|
|
344
|
-
|
|
1122
|
+
const viewportCtx = await browser.getViewportContext();
|
|
1123
|
+
// Screenshot gating: only capture on (a) initial page load — first step
|
|
1124
|
+
// ever for this URL, OR (b) when scroll position changed >100px since the
|
|
1125
|
+
// last capture. This avoids spamming the disk with near-identical frames
|
|
1126
|
+
// on tiny scroll/click steps and keeps the per-step paths renderer
|
|
1127
|
+
// limited to meaningful visual transitions.
|
|
1128
|
+
const screenshotKey = currentUrl.split('?')[0]; // strip query for stable keying
|
|
1129
|
+
const lastCapturedScrollY = lastScreenshotScroll.get(screenshotKey);
|
|
1130
|
+
const isInitialLoadForUrl = lastCapturedScrollY === undefined;
|
|
1131
|
+
const scrolledEnough = lastCapturedScrollY !== undefined && Math.abs(viewportCtx.scroll.y - lastCapturedScrollY) > 100;
|
|
1132
|
+
const shouldCapture = isInitialLoadForUrl || scrolledEnough;
|
|
1133
|
+
let screenshotPath;
|
|
1134
|
+
let attentionHeatmapBase64;
|
|
1135
|
+
if (shouldCapture) {
|
|
1136
|
+
// Live mode renders the heatmap IN the browser via canvas overlay
|
|
1137
|
+
// (see ATTENTION_OVERLAY_SCRIPT) so VNC picks it up natively. We just
|
|
1138
|
+
// poke the in-page recompute on capture triggers so it refreshes when
|
|
1139
|
+
// the page shape changes; the canvas already responds to scroll/resize
|
|
1140
|
+
// automatically. Server-side heatmap generation only runs for non-live
|
|
1141
|
+
// (or backwards-compat) callers.
|
|
1142
|
+
const screenshotP = browser.screenshot().catch(() => undefined);
|
|
1143
|
+
lastScreenshotScroll.set(screenshotKey, viewportCtx.scroll.y);
|
|
1144
|
+
if (options.enableMouseOverlay) {
|
|
1145
|
+
try {
|
|
1146
|
+
await page.evaluate(`window.__cbAttentionRefresh && window.__cbAttentionRefresh()`);
|
|
1147
|
+
}
|
|
1148
|
+
catch { /* page navigated mid-call */ }
|
|
1149
|
+
}
|
|
1150
|
+
// Wait for screenshot to finish (started in parallel above)
|
|
1151
|
+
screenshotPath = await screenshotP;
|
|
1152
|
+
}
|
|
1153
|
+
// Compute "what changed since last step" so the AI gets a structured diff
|
|
1154
|
+
// instead of having to re-derive it from raw state. Only meaningful from
|
|
1155
|
+
// step 2 onward (step 1 has no prior to compare against).
|
|
1156
|
+
if (lastActionOutcome) {
|
|
1157
|
+
const currentTexts = new Set(availableElements.map(e => e.text));
|
|
1158
|
+
const newClickables = [...currentTexts].filter(t => t && !prevClickableTexts.has(t)).slice(0, 8);
|
|
1159
|
+
const removedClickables = [...prevClickableTexts].filter(t => t && !currentTexts.has(t)).slice(0, 8);
|
|
1160
|
+
lastActionOutcome.newClickables = newClickables;
|
|
1161
|
+
lastActionOutcome.removedClickables = removedClickables;
|
|
1162
|
+
lastActionOutcome.scrollDelta = viewportCtx.scroll.y - prevScrollY;
|
|
1163
|
+
}
|
|
1164
|
+
prevClickableTexts = new Set(availableElements.map(e => e.text));
|
|
1165
|
+
prevScrollY = viewportCtx.scroll.y;
|
|
1166
|
+
// Extract visible page content. The old version grabbed the first 3 H1/H2/H3
|
|
1167
|
+
// and first 3 paragraphs by document order — meaning a Pricing page often
|
|
1168
|
+
// omitted the actual prices, and a long article omitted its body. The new
|
|
1169
|
+
// version walks the visible viewport and captures every meaningful text
|
|
1170
|
+
// node above-the-fold + a band just below, ordered top-to-bottom.
|
|
345
1171
|
const pageContent = await page.evaluate(() => {
|
|
346
|
-
const
|
|
347
|
-
//
|
|
348
|
-
|
|
1172
|
+
const vh = window.innerHeight;
|
|
1173
|
+
const lookahead = vh * 0.4; // include a strip just below the fold so AI can plan ahead
|
|
1174
|
+
const out = [];
|
|
1175
|
+
const skipTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'svg', 'IFRAME']);
|
|
1176
|
+
const nodes = document.querySelectorAll('h1, h2, h3, h4, p, li, td, th, [role="heading"], [data-testid]');
|
|
1177
|
+
nodes.forEach(el => {
|
|
1178
|
+
if (skipTags.has(el.tagName))
|
|
1179
|
+
return;
|
|
1180
|
+
const r = el.getBoundingClientRect();
|
|
1181
|
+
if (r.bottom < -50)
|
|
1182
|
+
return; // above viewport
|
|
1183
|
+
if (r.top > vh + lookahead)
|
|
1184
|
+
return; // far below
|
|
1185
|
+
if (r.width < 20 || r.height < 8)
|
|
1186
|
+
return;
|
|
1187
|
+
const style = window.getComputedStyle(el);
|
|
1188
|
+
if (style.display === 'none' || style.visibility === 'hidden')
|
|
1189
|
+
return;
|
|
349
1190
|
const text = el.innerText?.trim();
|
|
350
|
-
if (text
|
|
351
|
-
|
|
1191
|
+
if (!text || text.length < 3)
|
|
1192
|
+
return;
|
|
1193
|
+
// Trim long paragraphs but keep prices/key info
|
|
1194
|
+
const trimmed = text.length > 220 ? text.substring(0, 220) + '…' : text;
|
|
1195
|
+
const tag = el.tagName.toLowerCase();
|
|
1196
|
+
const tagLabel = ['h1', 'h2', 'h3', 'h4'].includes(tag) ? tag.toUpperCase() : tag;
|
|
1197
|
+
const visMark = r.top < vh && r.bottom > 0 ? '👁' : '↓';
|
|
1198
|
+
out.push({ y: r.top, line: `${visMark}[${tagLabel}] ${trimmed}` });
|
|
352
1199
|
});
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
1200
|
+
out.sort((a, b) => a.y - b.y);
|
|
1201
|
+
// Dedup adjacent identical lines (common in nested elements)
|
|
1202
|
+
const dedup = [];
|
|
1203
|
+
let prev = '';
|
|
1204
|
+
for (const o of out) {
|
|
1205
|
+
if (o.line !== prev) {
|
|
1206
|
+
dedup.push(o.line);
|
|
1207
|
+
prev = o.line;
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
return dedup.slice(0, 30).join('\n');
|
|
361
1211
|
}).catch(() => '');
|
|
362
1212
|
// Habituation/Banner Blindness tracking (v10.1.0)
|
|
363
1213
|
// Detect UI patterns in available elements and update habituation state
|
|
@@ -374,7 +1224,7 @@ export async function runCognitiveJourney(options) {
|
|
|
374
1224
|
}
|
|
375
1225
|
}
|
|
376
1226
|
// Build step prompt with available elements so Claude knows what's clickable and fillable
|
|
377
|
-
const stepPrompt = buildStepPrompt(state, currentUrl, pageTitle, step, availableElements, availableInputs, pageContent);
|
|
1227
|
+
const stepPrompt = buildStepPrompt(state, currentUrl, pageTitle, step, availableElements, availableInputs, pageContent, viewportCtx, lastActionOutcome);
|
|
378
1228
|
// Build message content - with or without vision
|
|
379
1229
|
let messageContent;
|
|
380
1230
|
if (options.vision && screenshotPath && existsSync(screenshotPath)) {
|
|
@@ -410,80 +1260,163 @@ export async function runCognitiveJourney(options) {
|
|
|
410
1260
|
messages.push({ role: "assistant", content: assistantMessage });
|
|
411
1261
|
// Parse Claude's response
|
|
412
1262
|
const parsed = parseCognitiveResponse(assistantMessage);
|
|
413
|
-
//
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
1263
|
+
// Capture URL BEFORE we apply any action. We use the previous URL +
|
|
1264
|
+
// the post-action URL to gate goal progress on real evidence — Claude's
|
|
1265
|
+
// self-reported progress is taken as a *signal*, but only allowed to
|
|
1266
|
+
// advance the canonical goalProgress if a concrete action actually
|
|
1267
|
+
// happened (URL changed, an interactive action succeeded, etc).
|
|
1268
|
+
const urlBeforeAction = currentUrl;
|
|
417
1269
|
state.currentMood = parsed.mood ?? state.currentMood;
|
|
418
|
-
//
|
|
419
|
-
|
|
420
|
-
//
|
|
421
|
-
//
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
//
|
|
1270
|
+
// ─── Single source of truth: drive everything through emotionalState ───
|
|
1271
|
+
//
|
|
1272
|
+
// Previously we wrote `state.confusionLevel = parsed.newConfusion`
|
|
1273
|
+
// and `state.frustrationLevel = parsed.newFrustration` directly,
|
|
1274
|
+
// running in parallel with the Scherer EmotionalState updates below.
|
|
1275
|
+
// Two systems, two truths, drift every step.
|
|
1276
|
+
//
|
|
1277
|
+
// Now: Claude's reported values are SIGNALS that fire emotional triggers
|
|
1278
|
+
// on the canonical EmotionalState. After the trigger applies and the
|
|
1279
|
+
// state decays (with persona-trait-modulated rates), we sync the
|
|
1280
|
+
// simplified-view fields (confusionLevel / frustrationLevel /
|
|
1281
|
+
// patienceRemaining) from the canonical store via deriveSimplifiedState.
|
|
1282
|
+
//
|
|
1283
|
+
// Consumers (worker, dashboard, MCP tools, persona-comparison) keep
|
|
1284
|
+
// reading the same field names; the values are just consistent now.
|
|
429
1285
|
if (state.emotionalState) {
|
|
430
|
-
// Determine
|
|
1286
|
+
// Determine triggers from the action outcome. Claude reports what
|
|
1287
|
+
// happened; we translate those signals into emotional triggers.
|
|
431
1288
|
const actionSuccess = parsed.actionSuccess !== false;
|
|
432
|
-
const
|
|
433
|
-
const
|
|
434
|
-
const
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
1289
|
+
const claudeConfusion = parsed.newConfusion ?? state.emotionalState.confusion;
|
|
1290
|
+
const claudeFrustration = parsed.newFrustration ?? state.emotionalState.frustration;
|
|
1291
|
+
const confusionIncreased = claudeConfusion > state.emotionalState.confusion + 0.05;
|
|
1292
|
+
const confusionDecreased = claudeConfusion < state.emotionalState.confusion - 0.05;
|
|
1293
|
+
const frustrationIncreased = claudeFrustration > state.emotionalState.frustration + 0.05;
|
|
1294
|
+
// Severity scales with the magnitude of the LLM-reported delta — Claude
|
|
1295
|
+
// proposes a value, we use the *delta* (clamped to [0, 1]) as the
|
|
1296
|
+
// trigger's intensity, instead of trusting Claude's absolute number.
|
|
1297
|
+
const confusionDelta = Math.max(0, Math.min(1, claudeConfusion - state.emotionalState.confusion));
|
|
1298
|
+
const frustrationDelta = Math.max(0, Math.min(1, claudeFrustration - state.emotionalState.frustration));
|
|
1299
|
+
// Triggers are evaluated in priority order; first match wins. Multiple
|
|
1300
|
+
// matched signals can fire sequentially below if both are strong.
|
|
1301
|
+
const triggersToFire = [];
|
|
438
1302
|
if (!actionSuccess && parsed.errorMessage) {
|
|
439
|
-
|
|
440
|
-
triggerDescription = parsed.errorMessage;
|
|
441
|
-
triggerSeverity = 1.2;
|
|
1303
|
+
triggersToFire.push({ trigger: "error", severity: 1.2, description: parsed.errorMessage });
|
|
442
1304
|
}
|
|
443
1305
|
else if (!actionSuccess) {
|
|
444
|
-
|
|
445
|
-
triggerDescription = "Action did not succeed";
|
|
1306
|
+
triggersToFire.push({ trigger: "failure", severity: 1.0, description: "Action did not succeed" });
|
|
446
1307
|
}
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
1308
|
+
// Progress is now an EVIDENCE-GATED trigger — see goalProgressDelta below.
|
|
1309
|
+
if (confusionIncreased) {
|
|
1310
|
+
triggersToFire.push({
|
|
1311
|
+
trigger: "confusion_onset",
|
|
1312
|
+
severity: Math.max(0.5, confusionDelta * 2),
|
|
1313
|
+
description: parsed.frictionDescription || "UI became confusing",
|
|
1314
|
+
});
|
|
450
1315
|
}
|
|
451
|
-
else if (
|
|
452
|
-
|
|
453
|
-
|
|
1316
|
+
else if (confusionDecreased) {
|
|
1317
|
+
// NEW trigger: when Claude reports clarity, fire the existing
|
|
1318
|
+
// "clarity" trigger — previously this only happened implicitly
|
|
1319
|
+
// via decay.
|
|
1320
|
+
triggersToFire.push({
|
|
1321
|
+
trigger: "clarity",
|
|
1322
|
+
severity: 0.8,
|
|
1323
|
+
description: "UI became clear",
|
|
1324
|
+
});
|
|
454
1325
|
}
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
1326
|
+
if (frustrationIncreased && actionSuccess) {
|
|
1327
|
+
// Friction without outright failure
|
|
1328
|
+
triggersToFire.push({
|
|
1329
|
+
trigger: "setback",
|
|
1330
|
+
severity: Math.max(0.5, frustrationDelta * 2),
|
|
1331
|
+
description: "Encountered friction",
|
|
1332
|
+
});
|
|
458
1333
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
1334
|
+
// Time pressure fires when patience is already derived-low (recompute
|
|
1335
|
+
// briefly here using current state for the gating check).
|
|
1336
|
+
const patiencePreview = deriveSimplifiedState(state.emotionalState, traits).patienceRemaining;
|
|
1337
|
+
if (patiencePreview < 0.3 && !triggersToFire.length) {
|
|
1338
|
+
triggersToFire.push({
|
|
1339
|
+
trigger: "time_pressure",
|
|
1340
|
+
severity: 0.8,
|
|
1341
|
+
description: "Running out of patience",
|
|
1342
|
+
});
|
|
463
1343
|
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
1344
|
+
if (actionSuccess && !triggersToFire.length) {
|
|
1345
|
+
triggersToFire.push({
|
|
1346
|
+
trigger: "success",
|
|
1347
|
+
severity: 0.8,
|
|
1348
|
+
description: parsed.action || "Action completed",
|
|
1349
|
+
});
|
|
468
1350
|
}
|
|
469
|
-
// Apply
|
|
470
|
-
|
|
471
|
-
const { state:
|
|
472
|
-
state.emotionalState =
|
|
1351
|
+
// Apply each trigger sequentially against the canonical state.
|
|
1352
|
+
for (const t of triggersToFire) {
|
|
1353
|
+
const { state: newEmotional, event } = applyEmotionalTrigger(state.emotionalState, t.trigger, emotionalConfig, state.stepCount, { severity: t.severity, description: t.description });
|
|
1354
|
+
state.emotionalState = newEmotional;
|
|
473
1355
|
state.emotionalJourney?.push(event);
|
|
474
1356
|
}
|
|
475
|
-
// Decay
|
|
476
|
-
|
|
477
|
-
//
|
|
1357
|
+
// Decay toward baseline using PERSONA-TRAIT-MODULATED rates:
|
|
1358
|
+
// comprehension speeds confusion recovery
|
|
1359
|
+
// resilience speeds frustration recovery
|
|
1360
|
+
// curiosity speeds boredom recovery
|
|
1361
|
+
// (Replaces the old uniform decayEmotions + the ad-hoc resilience-
|
|
1362
|
+
// applied-to-frustrationLevel block.)
|
|
1363
|
+
state.emotionalState = decayEmotionsWithTraits(state.emotionalState, emotionalConfig, traits);
|
|
1364
|
+
}
|
|
1365
|
+
// ─── Evidence-gated goal progress ───
|
|
1366
|
+
//
|
|
1367
|
+
// Claude proposes a goalProgress value. We only ACCEPT it if there's
|
|
1368
|
+
// concrete evidence the action actually advanced things:
|
|
1369
|
+
// - URL changed (we navigated somewhere new)
|
|
1370
|
+
// - the action was an interactive one (click/fill/scroll/wait) that succeeded
|
|
1371
|
+
// For in-page progress (URL unchanged but a successful interaction), we
|
|
1372
|
+
// allow at most +0.05 per step — small partial credit.
|
|
1373
|
+
//
|
|
1374
|
+
// This stops Claude's positivity bias from inflating progress without
|
|
1375
|
+
// any real navigation occurring.
|
|
1376
|
+
const llmProposedProgress = parsed.goalProgress ?? state.goalProgress;
|
|
1377
|
+
const progressDeltaProposed = llmProposedProgress - state.goalProgress;
|
|
1378
|
+
let progressDeltaAllowed = 0;
|
|
1379
|
+
if (progressDeltaProposed > 0) {
|
|
1380
|
+
const interactiveActions = new Set(["click", "fill", "navigate", "scroll", "select"]);
|
|
1381
|
+
const wasInteractive = parsed.action ? Array.from(interactiveActions).some(a => parsed.action.toLowerCase().startsWith(a)) : false;
|
|
1382
|
+
const actionSucceeded = parsed.actionSuccess !== false;
|
|
1383
|
+
// Note: urlChanged is finalized after executeAction below; we pre-allow
|
|
1384
|
+
// the LLM's delta if interactive+successful and reconcile post-action.
|
|
1385
|
+
if (wasInteractive && actionSucceeded) {
|
|
1386
|
+
// Allow up to the LLM's proposed delta; URL-change reconciliation later
|
|
1387
|
+
// expands the cap to full LLM proposal if URL changed.
|
|
1388
|
+
progressDeltaAllowed = Math.min(progressDeltaProposed, 0.05);
|
|
1389
|
+
}
|
|
1390
|
+
// No evidence at all → no progress (Claude was hallucinating advancement)
|
|
1391
|
+
}
|
|
1392
|
+
else {
|
|
1393
|
+
// Claude can lower progress freely — that's a backtrack signal.
|
|
1394
|
+
progressDeltaAllowed = progressDeltaProposed;
|
|
1395
|
+
}
|
|
1396
|
+
state.goalProgress = Math.max(0, Math.min(1, state.goalProgress + progressDeltaAllowed));
|
|
1397
|
+
// Stash the proposed progress so the post-action URL-change check can
|
|
1398
|
+
// expand the cap if a navigation actually happened.
|
|
1399
|
+
const _llmProposedProgressDelta = progressDeltaProposed;
|
|
1400
|
+
const _urlBeforeAction = urlBeforeAction;
|
|
1401
|
+
// ─── Sync derived simple-state fields from canonical EmotionalState ───
|
|
1402
|
+
if (state.emotionalState) {
|
|
1403
|
+
const derived = deriveSimplifiedState(state.emotionalState, traits);
|
|
1404
|
+
state.patienceRemaining = derived.patienceRemaining;
|
|
1405
|
+
state.confusionLevel = derived.confusionLevel;
|
|
1406
|
+
state.frustrationLevel = derived.frustrationLevel;
|
|
1407
|
+
// Verbose logging + abandonment heuristic remain useful — moved from
|
|
1408
|
+
// the old single big `if (state.emotionalState) { ... }` block that was
|
|
1409
|
+
// refactored above.
|
|
478
1410
|
if (options.verbose && state.emotionalState.dominant !== "neutral") {
|
|
479
1411
|
console.log(`💭 ${describeEmotionalState(state.emotionalState)}`);
|
|
480
1412
|
}
|
|
481
|
-
// Check if emotions suggest abandonment
|
|
482
1413
|
const abandonmentCheck = shouldConsiderAbandonment(state.emotionalState);
|
|
483
1414
|
if (abandonmentCheck.shouldConsider && options.verbose) {
|
|
484
1415
|
console.log(`⚠️ Emotional abandonment signal: ${abandonmentCheck.reason}`);
|
|
485
1416
|
}
|
|
486
1417
|
}
|
|
1418
|
+
void _llmProposedProgressDelta;
|
|
1419
|
+
void _urlBeforeAction;
|
|
487
1420
|
// Dual-Process Theory: System 1/2 switching (v10.0.0)
|
|
488
1421
|
if (state.cognitiveMode) {
|
|
489
1422
|
const stepStartTime = Date.now();
|
|
@@ -559,6 +1492,10 @@ export async function runCognitiveJourney(options) {
|
|
|
559
1492
|
confusion: state.confusionLevel,
|
|
560
1493
|
frustration: state.frustrationLevel,
|
|
561
1494
|
timestamp: Date.now(),
|
|
1495
|
+
screenshotPath,
|
|
1496
|
+
attentionHeatmapBase64,
|
|
1497
|
+
scrollY: viewportCtx.scroll.y,
|
|
1498
|
+
viewport: { width: viewportCtx.viewport.width, height: viewportCtx.viewport.height },
|
|
562
1499
|
});
|
|
563
1500
|
// Record friction point if confusion/frustration spiked
|
|
564
1501
|
if (parsed.frictionDescription && (state.confusionLevel > 0.4 || state.frustrationLevel > 0.4)) {
|
|
@@ -571,7 +1508,9 @@ export async function runCognitiveJourney(options) {
|
|
|
571
1508
|
monologue: parsed.monologue || "",
|
|
572
1509
|
});
|
|
573
1510
|
}
|
|
574
|
-
// Callback
|
|
1511
|
+
// Callback — emits BEFORE the action so the live viewer renders the
|
|
1512
|
+
// attention overlay first, then the persona acts. The dashboard receives
|
|
1513
|
+
// the heatmap base64 via SSE during this window.
|
|
575
1514
|
if (options.onStep) {
|
|
576
1515
|
options.onStep({
|
|
577
1516
|
step,
|
|
@@ -579,8 +1518,20 @@ export async function runCognitiveJourney(options) {
|
|
|
579
1518
|
monologue: parsed.monologue || "",
|
|
580
1519
|
action: parsed.action,
|
|
581
1520
|
state: { ...state },
|
|
1521
|
+
screenshotPath,
|
|
1522
|
+
attentionHeatmapBase64,
|
|
1523
|
+
scrollY: viewportCtx.scroll.y,
|
|
1524
|
+
viewport: { width: viewportCtx.viewport.width, height: viewportCtx.viewport.height },
|
|
1525
|
+
url: currentUrl,
|
|
582
1526
|
});
|
|
583
1527
|
}
|
|
1528
|
+
// Live viewer dwell — when a fresh attention map was computed this step,
|
|
1529
|
+
// hold briefly before executing the action so the viewer can register the
|
|
1530
|
+
// heatmap. 1.2s is enough to glance at it without making the journey feel
|
|
1531
|
+
// sluggish. Skipped on non-capture steps so the journey paces normally.
|
|
1532
|
+
if (options.enableMouseOverlay && attentionHeatmapBase64) {
|
|
1533
|
+
await new Promise((r) => setTimeout(r, 1200));
|
|
1534
|
+
}
|
|
584
1535
|
// Verbose output
|
|
585
1536
|
if (options.verbose) {
|
|
586
1537
|
console.log(`\n━━━ Step ${step} ━━━`);
|
|
@@ -632,31 +1583,175 @@ export async function runCognitiveJourney(options) {
|
|
|
632
1583
|
}
|
|
633
1584
|
// Execute action if provided
|
|
634
1585
|
if (parsed.action) {
|
|
1586
|
+
// Surface the action on-screen so anyone watching the live recording / VNC
|
|
1587
|
+
// can see what's about to happen. Best-effort — never block the action.
|
|
1588
|
+
if (options.enableMouseOverlay) {
|
|
1589
|
+
// Make sure overlay is present on the current page (a previous action
|
|
1590
|
+
// may have navigated and the init script raced our page.evaluate).
|
|
1591
|
+
await installMouseOverlay();
|
|
1592
|
+
try {
|
|
1593
|
+
const p = await browser.getPage();
|
|
1594
|
+
const label = parsed.actionTarget ? `${parsed.action} → ${parsed.actionTarget}` : parsed.action;
|
|
1595
|
+
await p.evaluate((t) => {
|
|
1596
|
+
// @ts-expect-error window helper installed by overlay script
|
|
1597
|
+
if (window.__cbShowAction)
|
|
1598
|
+
window.__cbShowAction(t);
|
|
1599
|
+
}, label.slice(0, 120));
|
|
1600
|
+
}
|
|
1601
|
+
catch { /* page navigated mid-call, helper will reinstall */ }
|
|
1602
|
+
}
|
|
635
1603
|
try {
|
|
636
|
-
const
|
|
1604
|
+
const indexMatch = /^click:#(\d+)$/.exec(parsed.action);
|
|
1605
|
+
const textMatch = /^click:(.+)$/.exec(parsed.action);
|
|
1606
|
+
let target;
|
|
1607
|
+
if (indexMatch) {
|
|
1608
|
+
target = availableElements[parseInt(indexMatch[1], 10)];
|
|
1609
|
+
}
|
|
1610
|
+
else if (textMatch && !textMatch[1].startsWith("#")) {
|
|
1611
|
+
// Find best element match by text. Prefer (1) exact match, then
|
|
1612
|
+
// (2) starts-with, then (3) includes; among ties, pick the one
|
|
1613
|
+
// closest to the current viewport center.
|
|
1614
|
+
const wanted = textMatch[1].trim().toLowerCase();
|
|
1615
|
+
const vc = viewportCtx.viewport.height / 2;
|
|
1616
|
+
const candidates = availableElements.filter(e => e.text);
|
|
1617
|
+
const score = (e) => {
|
|
1618
|
+
const t = (e.text || "").toLowerCase().trim();
|
|
1619
|
+
if (!t)
|
|
1620
|
+
return -Infinity;
|
|
1621
|
+
let base = 0;
|
|
1622
|
+
if (t === wanted)
|
|
1623
|
+
base = 1000;
|
|
1624
|
+
else if (t.startsWith(wanted) || wanted.startsWith(t))
|
|
1625
|
+
base = 500;
|
|
1626
|
+
else if (t.includes(wanted) || wanted.includes(t))
|
|
1627
|
+
base = 100;
|
|
1628
|
+
else
|
|
1629
|
+
return -Infinity;
|
|
1630
|
+
// Penalize distance from viewport center (closer = preferred)
|
|
1631
|
+
const ey = (e.y ?? 0) + (e.height ?? 0) / 2;
|
|
1632
|
+
const dist = Math.abs(ey - vc);
|
|
1633
|
+
return base - Math.min(dist, 5000);
|
|
1634
|
+
};
|
|
1635
|
+
let best;
|
|
1636
|
+
let bestScore = -Infinity;
|
|
1637
|
+
for (const e of candidates) {
|
|
1638
|
+
const s = score(e);
|
|
1639
|
+
if (s > bestScore) {
|
|
1640
|
+
bestScore = s;
|
|
1641
|
+
best = e;
|
|
1642
|
+
}
|
|
1643
|
+
}
|
|
1644
|
+
if (best && bestScore > 0)
|
|
1645
|
+
target = best;
|
|
1646
|
+
}
|
|
1647
|
+
let result;
|
|
1648
|
+
const hasBbox = target && typeof target.x === "number" && typeof target.y === "number"
|
|
1649
|
+
&& typeof target.width === "number" && typeof target.height === "number";
|
|
1650
|
+
if (hasBbox && target) {
|
|
1651
|
+
const page = await browser.getPage();
|
|
1652
|
+
const urlBefore = page.url();
|
|
1653
|
+
const cx = target.x + target.width / 2;
|
|
1654
|
+
const cy = target.y + target.height / 2;
|
|
1655
|
+
try {
|
|
1656
|
+
// Only scroll if the element is OFF-screen. Scrolling an already-
|
|
1657
|
+
// visible element causes pointless jitter that stacks with any
|
|
1658
|
+
// page-level smooth-scroll handler firing on click.
|
|
1659
|
+
const inView = cy >= 0 && cy <= viewportCtx.viewport.height;
|
|
1660
|
+
if (!inView) {
|
|
1661
|
+
await page.evaluate(({ y }) => {
|
|
1662
|
+
const docTop = window.scrollY + y;
|
|
1663
|
+
const t = Math.max(0, docTop - window.innerHeight / 2);
|
|
1664
|
+
// "instant" behavior — no animation, no stacking with page handlers
|
|
1665
|
+
window.scrollTo({ top: t, behavior: "instant" });
|
|
1666
|
+
}, { y: cy });
|
|
1667
|
+
await page.waitForTimeout(150);
|
|
1668
|
+
}
|
|
1669
|
+
// Re-read post-scroll coords; click at center via raw mouse so
|
|
1670
|
+
// Playwright's locator auto-scroll-into-view never engages.
|
|
1671
|
+
const fresh = await page.evaluate(({ sel }) => {
|
|
1672
|
+
const el = document.querySelector(sel);
|
|
1673
|
+
if (!el)
|
|
1674
|
+
return null;
|
|
1675
|
+
const r = el.getBoundingClientRect();
|
|
1676
|
+
return { x: r.left + r.width / 2, y: r.top + r.height / 2, w: r.width, h: r.height };
|
|
1677
|
+
}, { sel: target.selector });
|
|
1678
|
+
const ctr = (fresh && fresh.x > 0 && fresh.y > 0)
|
|
1679
|
+
? { x: fresh.x, y: fresh.y, w: fresh.w, h: fresh.h }
|
|
1680
|
+
: { x: cx, y: cy, w: target.width, h: target.height };
|
|
1681
|
+
if (options.humanInput) {
|
|
1682
|
+
const { humanClick } = await import("../visual/human-input.js");
|
|
1683
|
+
await humanClick(page, ctr.x, ctr.y, inputTraits, { width: ctr.w, height: ctr.h });
|
|
1684
|
+
}
|
|
1685
|
+
else {
|
|
1686
|
+
await page.mouse.click(ctr.x, ctr.y);
|
|
1687
|
+
}
|
|
1688
|
+
await page.waitForTimeout(800);
|
|
1689
|
+
const urlAfter = page.url();
|
|
1690
|
+
result = { success: true, ...(urlAfter !== urlBefore ? { newUrl: urlAfter } : {}) };
|
|
1691
|
+
}
|
|
1692
|
+
catch (clickErr) {
|
|
1693
|
+
console.warn(`[cognitive] coordinate click failed: ${clickErr.message}`);
|
|
1694
|
+
const fallback = target.text ? `click:${target.text}` : parsed.action;
|
|
1695
|
+
result = await executeAction(browser, fallback, fittsParams, state.gazeMouseLag, options.humanInput, inputTraits);
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
else {
|
|
1699
|
+
// No bbox — let executeAction handle (non-click actions, or click
|
|
1700
|
+
// text not found in availableElements).
|
|
1701
|
+
result = await executeAction(browser, parsed.action, fittsParams, state.gazeMouseLag, options.humanInput, inputTraits);
|
|
1702
|
+
}
|
|
1703
|
+
// Inter-action settle pause — humans don't fire actions back-to-back
|
|
1704
|
+
// at machine speed. Modulated by patience + readingTendency so an
|
|
1705
|
+
// impatient persona snaps through faster than a thorough one.
|
|
1706
|
+
if (options.humanInput) {
|
|
1707
|
+
const { humanSettle } = await import("../visual/human-input.js");
|
|
1708
|
+
await humanSettle(inputTraits);
|
|
1709
|
+
}
|
|
637
1710
|
state.memory.actionsAttempted.push({
|
|
638
1711
|
action: parsed.action,
|
|
639
1712
|
target: parsed.actionTarget,
|
|
640
1713
|
success: result.success,
|
|
641
1714
|
});
|
|
642
1715
|
// Track page visits if URL changed
|
|
1716
|
+
const urlChanged = !!result.newUrl && result.newUrl !== _urlBeforeAction;
|
|
1717
|
+
// Record the outcome for the next step's prompt. The diff fields
|
|
1718
|
+
// (newClickables, removedClickables, scrollDelta) are filled in at
|
|
1719
|
+
// the top of step N+1 by comparing the fresh perception to the cache.
|
|
1720
|
+
lastActionOutcome = {
|
|
1721
|
+
action: parsed.action,
|
|
1722
|
+
target: parsed.actionTarget,
|
|
1723
|
+
success: result.success,
|
|
1724
|
+
urlChanged,
|
|
1725
|
+
fromUrl: _urlBeforeAction,
|
|
1726
|
+
toUrl: result.newUrl ?? _urlBeforeAction,
|
|
1727
|
+
};
|
|
643
1728
|
if (result.newUrl && !state.memory.pagesVisited.includes(result.newUrl)) {
|
|
644
1729
|
state.memory.pagesVisited.push(result.newUrl);
|
|
645
1730
|
currentUrl = result.newUrl;
|
|
646
1731
|
}
|
|
647
|
-
//
|
|
648
|
-
//
|
|
649
|
-
//
|
|
650
|
-
//
|
|
651
|
-
if (
|
|
652
|
-
const
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
1732
|
+
// ─── Post-action evidence reconciliation ───
|
|
1733
|
+
// Goal progress cap was previously +0.05 max (no-evidence fallback).
|
|
1734
|
+
// If the URL actually changed, expand the cap to the LLM's full
|
|
1735
|
+
// proposed delta — that's strong navigation evidence.
|
|
1736
|
+
if (urlChanged && _llmProposedProgressDelta > 0.05) {
|
|
1737
|
+
const additionalAllowed = _llmProposedProgressDelta - 0.05;
|
|
1738
|
+
state.goalProgress = Math.min(1, state.goalProgress + additionalAllowed);
|
|
1739
|
+
}
|
|
1740
|
+
// Action result feeds back into the canonical EmotionalState via triggers.
|
|
1741
|
+
// Replaces the prior direct writes to state.frustrationLevel / patienceRemaining,
|
|
1742
|
+
// which were overwritten next step by deriveSimplifiedState anyway.
|
|
1743
|
+
if (state.emotionalState) {
|
|
1744
|
+
if (result.success) {
|
|
1745
|
+
const progressMade = urlChanged || state.goalProgress > 0.1;
|
|
1746
|
+
// Progress = stronger positive trigger than plain success
|
|
1747
|
+
const { state: newEmotional, event } = applyEmotionalTrigger(state.emotionalState, progressMade ? "progress" : "success", emotionalConfig, state.stepCount, {
|
|
1748
|
+
severity: progressMade ? 1.2 : 0.8,
|
|
1749
|
+
description: progressMade
|
|
1750
|
+
? `Action advanced goal (${urlChanged ? "URL change" : "in-page progress"})`
|
|
1751
|
+
: `Action completed: ${parsed.action}`,
|
|
1752
|
+
});
|
|
1753
|
+
state.emotionalState = newEmotional;
|
|
1754
|
+
state.emotionalJourney?.push(event);
|
|
660
1755
|
}
|
|
661
1756
|
}
|
|
662
1757
|
}
|
|
@@ -665,7 +1760,32 @@ export async function runCognitiveJourney(options) {
|
|
|
665
1760
|
error: error instanceof Error ? error.message : String(error),
|
|
666
1761
|
context: `Step ${step}: ${parsed.action}`,
|
|
667
1762
|
});
|
|
668
|
-
|
|
1763
|
+
lastActionOutcome = {
|
|
1764
|
+
action: parsed.action,
|
|
1765
|
+
target: parsed.actionTarget,
|
|
1766
|
+
success: false,
|
|
1767
|
+
urlChanged: false,
|
|
1768
|
+
fromUrl: _urlBeforeAction,
|
|
1769
|
+
};
|
|
1770
|
+
// Action threw — fire an error trigger on canonical state.
|
|
1771
|
+
if (state.emotionalState) {
|
|
1772
|
+
const { state: newEmotional, event } = applyEmotionalTrigger(state.emotionalState, "error", emotionalConfig, state.stepCount, {
|
|
1773
|
+
severity: 1.3,
|
|
1774
|
+
description: error instanceof Error ? error.message : String(error),
|
|
1775
|
+
});
|
|
1776
|
+
state.emotionalState = newEmotional;
|
|
1777
|
+
state.emotionalJourney?.push(event);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
// Re-sync derived simple-state fields from canonical state after
|
|
1781
|
+
// post-action triggers have applied. Consumers that read
|
|
1782
|
+
// state.patienceRemaining / confusionLevel / frustrationLevel see the
|
|
1783
|
+
// up-to-date derived values.
|
|
1784
|
+
if (state.emotionalState) {
|
|
1785
|
+
const derived = deriveSimplifiedState(state.emotionalState, traits);
|
|
1786
|
+
state.patienceRemaining = derived.patienceRemaining;
|
|
1787
|
+
state.confusionLevel = derived.confusionLevel;
|
|
1788
|
+
state.frustrationLevel = derived.frustrationLevel;
|
|
669
1789
|
}
|
|
670
1790
|
// Update decision fatigue (v9.9.0)
|
|
671
1791
|
// Every action is a decision - fatigue accumulates
|
|
@@ -868,13 +1988,32 @@ BEHAVIOR GUIDELINES:
|
|
|
868
1988
|
|
|
869
1989
|
Always respond with valid JSON.`;
|
|
870
1990
|
}
|
|
871
|
-
function buildStepPrompt(state, currentUrl, pageTitle, step, availableElements = [], availableInputs = [], pageContent = "") {
|
|
1991
|
+
function buildStepPrompt(state, currentUrl, pageTitle, step, availableElements = [], availableInputs = [], pageContent = "", viewportCtx, lastOutcome) {
|
|
1992
|
+
// Group clickables by region so the AI can disambiguate "Pricing" in nav vs hero vs footer.
|
|
1993
|
+
// Index each element so the AI can use [#3] style references when text isn't unique.
|
|
872
1994
|
const elementsStr = availableElements.length > 0
|
|
873
|
-
?
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1995
|
+
? (() => {
|
|
1996
|
+
const groups = {};
|
|
1997
|
+
availableElements.forEach((e, i) => {
|
|
1998
|
+
const region = e.region ?? 'unknown';
|
|
1999
|
+
const aboveFold = e.aboveFold;
|
|
2000
|
+
const x = e.x;
|
|
2001
|
+
const y = e.y;
|
|
2002
|
+
const visMark = aboveFold ? '👁' : '↓';
|
|
2003
|
+
const posStr = (typeof x === 'number' && typeof y === 'number') ? ` @(${x},${y})` : '';
|
|
2004
|
+
const line = ` [#${i}] ${visMark} "${e.text}" (${e.tag}${e.role ? `, role=${e.role}` : ''}${posStr})`;
|
|
2005
|
+
(groups[region] ??= []).push(line);
|
|
2006
|
+
});
|
|
2007
|
+
const order = ['header-nav', 'top-bar', 'modal', 'main-visible', 'form', 'sidebar', 'below-fold', 'footer', 'unknown'];
|
|
2008
|
+
const out = [];
|
|
2009
|
+
for (const r of order) {
|
|
2010
|
+
if (groups[r]?.length) {
|
|
2011
|
+
out.push(` — ${r.toUpperCase()} —`);
|
|
2012
|
+
out.push(...groups[r]);
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
return out.join('\n');
|
|
2016
|
+
})()
|
|
878
2017
|
: " (no clickable elements detected)";
|
|
879
2018
|
// Format inputs - highlight hidden ones with their triggers, show options for selects
|
|
880
2019
|
const inputsStr = availableInputs.length > 0
|
|
@@ -896,13 +2035,58 @@ function buildStepPrompt(state, currentUrl, pageTitle, step, availableElements =
|
|
|
896
2035
|
const contentStr = pageContent
|
|
897
2036
|
? `\nVISIBLE PAGE CONTENT:\n${pageContent}\n`
|
|
898
2037
|
: "";
|
|
2038
|
+
// Viewport / scroll / overlay summary — gives the AI ground truth about what
|
|
2039
|
+
// is on screen NOW. Without this it has no idea if it just scrolled.
|
|
2040
|
+
let viewportStr = "";
|
|
2041
|
+
if (viewportCtx) {
|
|
2042
|
+
const { viewport, scroll, headings, activeModals, formErrors, bannersBlocking } = viewportCtx;
|
|
2043
|
+
const visHeadings = headings.filter(h => h.visible).slice(0, 6).map(h => `H${h.level}: ${h.text}`).join(' | ');
|
|
2044
|
+
const lines = [
|
|
2045
|
+
`- Viewport: ${viewport.width}×${viewport.height}`,
|
|
2046
|
+
`- Scroll: ${scroll.y}/${scroll.max}px (${Math.round(scroll.pct * 100)}%)${scroll.atTop ? ' [at top]' : ''}${scroll.atBottom ? ' [at bottom]' : ''}`,
|
|
2047
|
+
];
|
|
2048
|
+
if (visHeadings)
|
|
2049
|
+
lines.push(`- Currently visible headings: ${visHeadings}`);
|
|
2050
|
+
if (activeModals > 0)
|
|
2051
|
+
lines.push(`- ⚠️ ${activeModals} active modal/dialog blocking interaction — try dismiss first`);
|
|
2052
|
+
if (bannersBlocking.length > 0)
|
|
2053
|
+
lines.push(`- ⚠️ Banner blocking content: "${bannersBlocking[0]}" — try dismiss`);
|
|
2054
|
+
if (formErrors.length > 0)
|
|
2055
|
+
lines.push(`- ⚠️ Form errors visible: ${formErrors.join(' | ')}`);
|
|
2056
|
+
viewportStr = `\nVIEWPORT STATE:\n${lines.join('\n')}\n`;
|
|
2057
|
+
}
|
|
2058
|
+
// Action outcome — what actually happened from your last decision. Critical
|
|
2059
|
+
// for grounding the AI's monologue in reality.
|
|
2060
|
+
let outcomeStr = "";
|
|
2061
|
+
if (lastOutcome) {
|
|
2062
|
+
const lines = [];
|
|
2063
|
+
const target = lastOutcome.target ? ` → ${lastOutcome.target}` : '';
|
|
2064
|
+
lines.push(`- Last action: ${lastOutcome.action}${target} — ${lastOutcome.success ? '✓ executed' : '✗ failed'}`);
|
|
2065
|
+
if (lastOutcome.urlChanged) {
|
|
2066
|
+
lines.push(`- Page navigated: ${lastOutcome.fromUrl} → ${lastOutcome.toUrl}`);
|
|
2067
|
+
}
|
|
2068
|
+
else if (lastOutcome.action.startsWith('click')) {
|
|
2069
|
+
lines.push(`- Page did NOT navigate (click had no nav effect — element may be JS-only or wasn't matched)`);
|
|
2070
|
+
}
|
|
2071
|
+
if (typeof lastOutcome.scrollDelta === 'number' && Math.abs(lastOutcome.scrollDelta) > 10) {
|
|
2072
|
+
const dir = lastOutcome.scrollDelta > 0 ? 'down' : 'up';
|
|
2073
|
+
lines.push(`- Scrolled ${dir} ${Math.abs(lastOutcome.scrollDelta)}px`);
|
|
2074
|
+
}
|
|
2075
|
+
if (lastOutcome.newClickables?.length) {
|
|
2076
|
+
lines.push(`- New elements appeared: ${lastOutcome.newClickables.slice(0, 5).map(t => `"${t}"`).join(', ')}`);
|
|
2077
|
+
}
|
|
2078
|
+
if (lastOutcome.removedClickables?.length) {
|
|
2079
|
+
lines.push(`- Elements no longer visible: ${lastOutcome.removedClickables.slice(0, 5).map(t => `"${t}"`).join(', ')}`);
|
|
2080
|
+
}
|
|
2081
|
+
outcomeStr = `\nWHAT HAPPENED LAST STEP:\n${lines.join('\n')}\n`;
|
|
2082
|
+
}
|
|
899
2083
|
return `STEP ${step}
|
|
900
2084
|
|
|
901
2085
|
CURRENT PAGE:
|
|
902
2086
|
- URL: ${currentUrl}
|
|
903
2087
|
- Title: ${pageTitle}
|
|
904
|
-
${contentStr}
|
|
905
|
-
AVAILABLE ELEMENTS (clickable):
|
|
2088
|
+
${viewportStr}${outcomeStr}${contentStr}
|
|
2089
|
+
AVAILABLE ELEMENTS (clickable, grouped by region; 👁=above-fold, ↓=below-fold):
|
|
906
2090
|
${elementsStr}
|
|
907
2091
|
|
|
908
2092
|
FORM INPUTS (fillable):
|
|
@@ -916,11 +2100,25 @@ CURRENT STATE:
|
|
|
916
2100
|
- Mood: ${state.currentMood}
|
|
917
2101
|
- Pages Visited: ${state.memory.pagesVisited.length}
|
|
918
2102
|
- Actions Attempted: ${state.memory.actionsAttempted.length}
|
|
2103
|
+
${(() => {
|
|
2104
|
+
// Loop detection: if the last 4 actions are alternating scrolls, surface
|
|
2105
|
+
// it loudly so the AI breaks out instead of pumping scroll:down/up forever.
|
|
2106
|
+
const recent = state.memory.actionsAttempted.slice(-4).map(a => a.action || "");
|
|
2107
|
+
if (recent.length >= 4 && recent.every(a => a.startsWith("scroll:"))) {
|
|
2108
|
+
return `\n⚠️ LOOP DETECTED: your last 4 actions were all scrolls (${recent.join(", ")}). Pick a click/back/dismiss/find action instead — scrolling more will not help.`;
|
|
2109
|
+
}
|
|
2110
|
+
// Repeated failed actions
|
|
2111
|
+
const lastTwo = state.memory.actionsAttempted.slice(-2);
|
|
2112
|
+
if (lastTwo.length === 2 && lastTwo.every(a => a.success === false)) {
|
|
2113
|
+
return `\n⚠️ Last 2 actions failed (${lastTwo.map(a => a.action).join(", ")}). The element you wanted may not exist with that exact text — try a click:#N index from AVAILABLE ELEMENTS above, or a different element entirely.`;
|
|
2114
|
+
}
|
|
2115
|
+
return "";
|
|
2116
|
+
})()}
|
|
919
2117
|
|
|
920
|
-
|
|
2118
|
+
Ground your monologue in WHAT HAPPENED LAST STEP and VIEWPORT STATE — don't pretend you see things that aren't in AVAILABLE ELEMENTS or VISIBLE PAGE CONTENT. If the URL did not change after a click, say so explicitly.
|
|
921
2119
|
|
|
922
2120
|
ACTIONS (choose one):
|
|
923
|
-
- click:ElementText — click a link or button (use exact text from AVAILABLE ELEMENTS)
|
|
2121
|
+
- click:ElementText — click a link or button (use exact text from AVAILABLE ELEMENTS; for ambiguous text use click:#N where N is the [#N] index)
|
|
924
2122
|
- fill:FieldName:value — type into a form field
|
|
925
2123
|
- scroll:down — scroll down to see more content (TRY THIS FIRST if goal not visible)
|
|
926
2124
|
- scroll:up — scroll back up
|
|
@@ -936,11 +2134,12 @@ ACTIONS (choose one):
|
|
|
936
2134
|
- wait:ms — wait for content to load
|
|
937
2135
|
|
|
938
2136
|
STRATEGY:
|
|
939
|
-
1. If
|
|
940
|
-
2. If
|
|
941
|
-
3.
|
|
942
|
-
4.
|
|
943
|
-
5.
|
|
2137
|
+
1. If a modal/banner is blocking, dismiss FIRST
|
|
2138
|
+
2. If you don't see what you need, scroll:down — most content is below the fold
|
|
2139
|
+
3. If a click didn't navigate when you expected it to, the element may have a different selector — try click:#N with the index, or pick a different element
|
|
2140
|
+
4. If scrolling doesn't help, try find:keyword to search the page
|
|
2141
|
+
5. Use back if you navigated to the wrong page
|
|
2142
|
+
6. Only abandon after trying scroll + find + back
|
|
944
2143
|
|
|
945
2144
|
Respond in JSON format.`;
|
|
946
2145
|
}
|
|
@@ -1048,7 +2247,18 @@ function checkAbandonmentTriggers(state, thresholds) {
|
|
|
1048
2247
|
* For cognitive simulation, we estimate distance as 300px (average screen movement)
|
|
1049
2248
|
* and target width as 80px (average button size). Persona traits modify timing.
|
|
1050
2249
|
*/
|
|
1051
|
-
|
|
2250
|
+
function ordinalSuffix(n) {
|
|
2251
|
+
const tens = n % 100;
|
|
2252
|
+
if (tens >= 11 && tens <= 13)
|
|
2253
|
+
return "th";
|
|
2254
|
+
switch (n % 10) {
|
|
2255
|
+
case 1: return "st";
|
|
2256
|
+
case 2: return "nd";
|
|
2257
|
+
case 3: return "rd";
|
|
2258
|
+
default: return "th";
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
async function executeAction(browser, action, fittsParams, gazeMouseLag, humanInput, inputTraits) {
|
|
1052
2262
|
const [type, ...args] = action.split(":");
|
|
1053
2263
|
// Apply Fitts' Law timing for mouse actions (v9.9.0)
|
|
1054
2264
|
let movementTimeMs = 0;
|
|
@@ -1063,8 +2273,12 @@ async function executeAction(browser, action, fittsParams, gazeMouseLag) {
|
|
|
1063
2273
|
if (gazeMouseLag) {
|
|
1064
2274
|
movementTimeMs += gazeMouseLag;
|
|
1065
2275
|
}
|
|
1066
|
-
//
|
|
1067
|
-
|
|
2276
|
+
// When humanInput is enabled, humanClick provides its own hover-and-dwell
|
|
2277
|
+
// pacing — skip the pre-action sleep here so they don't compound (which
|
|
2278
|
+
// would make live mode slower than non-live, defeating the purpose).
|
|
2279
|
+
if (!humanInput) {
|
|
2280
|
+
await sleep(movementTimeMs);
|
|
2281
|
+
}
|
|
1068
2282
|
}
|
|
1069
2283
|
switch (type) {
|
|
1070
2284
|
case "click": {
|
|
@@ -1138,6 +2352,33 @@ async function executeAction(browser, action, fittsParams, gazeMouseLag) {
|
|
|
1138
2352
|
case "fill": {
|
|
1139
2353
|
const [selector, ...valueParts] = args;
|
|
1140
2354
|
const value = valueParts.join(":");
|
|
2355
|
+
if (humanInput) {
|
|
2356
|
+
// Human-realistic typing: focus the field, then type char-by-char
|
|
2357
|
+
// with Gaussian timing, occasional typos, and thinking pauses.
|
|
2358
|
+
const page = await browser.getPage();
|
|
2359
|
+
try {
|
|
2360
|
+
// Find the field by selector or label/placeholder
|
|
2361
|
+
const loc = page.locator(selector).first();
|
|
2362
|
+
if (await loc.count() === 0) {
|
|
2363
|
+
// Fallback to fill which has fuzzier matching
|
|
2364
|
+
const r = await browser.fill(selector, value);
|
|
2365
|
+
return { success: r.success };
|
|
2366
|
+
}
|
|
2367
|
+
await loc.scrollIntoViewIfNeeded({ timeout: 2000 }).catch(() => { });
|
|
2368
|
+
await loc.click({ timeout: 3000 });
|
|
2369
|
+
// Clear if there's existing content
|
|
2370
|
+
await page.keyboard.press("Control+A").catch(() => { });
|
|
2371
|
+
await page.keyboard.press("Delete").catch(() => { });
|
|
2372
|
+
const { humanType } = await import("../visual/human-input.js");
|
|
2373
|
+
await humanType(page, value, inputTraits);
|
|
2374
|
+
return { success: true };
|
|
2375
|
+
}
|
|
2376
|
+
catch (err) {
|
|
2377
|
+
// Fall back to instant fill if anything goes wrong
|
|
2378
|
+
const r = await browser.fill(selector, value);
|
|
2379
|
+
return { success: r.success };
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
1141
2382
|
// Apply KLM timing for typing (v9.10.0)
|
|
1142
2383
|
// Use age modifier as proxy for typing expertise (younger = faster)
|
|
1143
2384
|
const expertise = fittsParams?.ageModifier
|
|
@@ -1160,24 +2401,37 @@ async function executeAction(browser, action, fittsParams, gazeMouseLag) {
|
|
|
1160
2401
|
const direction = args[0]?.toLowerCase() || "down";
|
|
1161
2402
|
const page = await browser.getPage();
|
|
1162
2403
|
try {
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
break;
|
|
1170
|
-
case "up":
|
|
1171
|
-
await page.evaluate(() => window.scrollBy({ top: -400, behavior: "smooth" }));
|
|
1172
|
-
break;
|
|
1173
|
-
case "down":
|
|
1174
|
-
default:
|
|
1175
|
-
await page.evaluate(() => window.scrollBy({ top: 400, behavior: "smooth" }));
|
|
1176
|
-
break;
|
|
2404
|
+
const beforeY = await page.evaluate(() => window.scrollY);
|
|
2405
|
+
if (humanInput && (direction === "down" || direction === "up")) {
|
|
2406
|
+
// Human-realistic scroll: variable delta + occasional glance-back.
|
|
2407
|
+
const { humanScroll } = await import("../visual/human-input.js");
|
|
2408
|
+
const baseDelta = direction === "down" ? 400 : -400;
|
|
2409
|
+
await humanScroll(page, baseDelta, inputTraits);
|
|
1177
2410
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
2411
|
+
else {
|
|
2412
|
+
switch (direction) {
|
|
2413
|
+
case "top":
|
|
2414
|
+
await page.evaluate(() => window.scrollTo({ top: 0, behavior: "smooth" }));
|
|
2415
|
+
break;
|
|
2416
|
+
case "bottom":
|
|
2417
|
+
await page.evaluate(() => window.scrollTo({ top: document.body.scrollHeight, behavior: "smooth" }));
|
|
2418
|
+
break;
|
|
2419
|
+
case "up":
|
|
2420
|
+
await page.evaluate(() => window.scrollBy({ top: -400, behavior: "smooth" }));
|
|
2421
|
+
break;
|
|
2422
|
+
case "down":
|
|
2423
|
+
default:
|
|
2424
|
+
await page.evaluate(() => window.scrollBy({ top: 400, behavior: "smooth" }));
|
|
2425
|
+
break;
|
|
2426
|
+
}
|
|
2427
|
+
await sleep(300);
|
|
2428
|
+
}
|
|
2429
|
+
// Report success only if the page actually moved. A no-op scroll
|
|
2430
|
+
// (already at top/bottom, or scroll blocked) returns failure so the
|
|
2431
|
+
// AI knows to try something else instead of looping scroll:up/down.
|
|
2432
|
+
const afterY = await page.evaluate(() => window.scrollY);
|
|
2433
|
+
const moved = Math.abs(afterY - beforeY) > 5;
|
|
2434
|
+
return { success: moved };
|
|
1181
2435
|
}
|
|
1182
2436
|
catch {
|
|
1183
2437
|
return { success: false };
|