donobu 5.7.0 → 5.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/dist/assets/control-panel-tbd.html +530 -0
  2. package/dist/esm/assets/control-panel-tbd.html +530 -0
  3. package/dist/esm/lib/ai/PageAi.d.ts.map +1 -1
  4. package/dist/esm/lib/ai/PageAi.js +3 -1
  5. package/dist/esm/lib/ai/PageAi.js.map +1 -1
  6. package/dist/esm/lib/page/DonobuExtendedPage.d.ts +35 -0
  7. package/dist/esm/lib/page/DonobuExtendedPage.d.ts.map +1 -1
  8. package/dist/esm/lib/page/extendPage.d.ts +2 -0
  9. package/dist/esm/lib/page/extendPage.d.ts.map +1 -1
  10. package/dist/esm/lib/page/extendPage.js +7 -0
  11. package/dist/esm/lib/page/extendPage.js.map +1 -1
  12. package/dist/esm/lib/page/index.d.ts +1 -0
  13. package/dist/esm/lib/page/index.d.ts.map +1 -1
  14. package/dist/esm/lib/page/index.js +3 -0
  15. package/dist/esm/lib/page/index.js.map +1 -1
  16. package/dist/esm/lib/page/tbd/actionRecorder.d.ts +31 -0
  17. package/dist/esm/lib/page/tbd/actionRecorder.d.ts.map +1 -0
  18. package/dist/esm/lib/page/tbd/actionRecorder.js +85 -0
  19. package/dist/esm/lib/page/tbd/actionRecorder.js.map +1 -0
  20. package/dist/esm/lib/page/tbd/callSiteCapture.d.ts +21 -0
  21. package/dist/esm/lib/page/tbd/callSiteCapture.d.ts.map +1 -0
  22. package/dist/esm/lib/page/tbd/callSiteCapture.js +85 -0
  23. package/dist/esm/lib/page/tbd/callSiteCapture.js.map +1 -0
  24. package/dist/esm/lib/page/tbd/fileRewriter.d.ts +10 -0
  25. package/dist/esm/lib/page/tbd/fileRewriter.d.ts.map +1 -0
  26. package/dist/esm/lib/page/tbd/fileRewriter.js +174 -0
  27. package/dist/esm/lib/page/tbd/fileRewriter.js.map +1 -0
  28. package/dist/esm/lib/page/tbd/tbdTypes.d.ts +23 -0
  29. package/dist/esm/lib/page/tbd/tbdTypes.d.ts.map +1 -0
  30. package/dist/esm/lib/page/tbd/tbdTypes.js +3 -0
  31. package/dist/esm/lib/page/tbd/tbdTypes.js.map +1 -0
  32. package/dist/esm/lib/page/tbd.d.ts +19 -0
  33. package/dist/esm/lib/page/tbd.d.ts.map +1 -0
  34. package/dist/esm/lib/page/tbd.js +378 -0
  35. package/dist/esm/lib/page/tbd.js.map +1 -0
  36. package/dist/esm/lib/test/testExtension.d.ts.map +1 -1
  37. package/dist/esm/lib/test/testExtension.js +15 -0
  38. package/dist/esm/lib/test/testExtension.js.map +1 -1
  39. package/dist/esm/managers/EnvDataManager.d.ts +13 -1
  40. package/dist/esm/managers/EnvDataManager.d.ts.map +1 -1
  41. package/dist/esm/managers/EnvDataManager.js +13 -1
  42. package/dist/esm/managers/EnvDataManager.js.map +1 -1
  43. package/dist/esm/persistence/env/EnvPersistence.d.ts +20 -0
  44. package/dist/esm/persistence/env/EnvPersistence.d.ts.map +1 -1
  45. package/dist/esm/persistence/env/EnvPersistenceRegistry.d.ts +9 -1
  46. package/dist/esm/persistence/env/EnvPersistenceRegistry.d.ts.map +1 -1
  47. package/dist/esm/persistence/env/EnvPersistenceRegistry.js +1 -1
  48. package/dist/esm/persistence/env/EnvPersistenceRegistry.js.map +1 -1
  49. package/dist/esm/persistence/env/EnvPersistenceSqlite.d.ts +5 -1
  50. package/dist/esm/persistence/env/EnvPersistenceSqlite.d.ts.map +1 -1
  51. package/dist/esm/persistence/env/EnvPersistenceSqlite.js +5 -1
  52. package/dist/esm/persistence/env/EnvPersistenceSqlite.js.map +1 -1
  53. package/dist/esm/persistence/env/EnvPersistenceVolatile.d.ts +8 -1
  54. package/dist/esm/persistence/env/EnvPersistenceVolatile.d.ts.map +1 -1
  55. package/dist/esm/persistence/env/EnvPersistenceVolatile.js +8 -1
  56. package/dist/esm/persistence/env/EnvPersistenceVolatile.js.map +1 -1
  57. package/dist/esm/utils/MiscUtils.d.ts +5 -0
  58. package/dist/esm/utils/MiscUtils.d.ts.map +1 -1
  59. package/dist/esm/utils/MiscUtils.js +12 -5
  60. package/dist/esm/utils/MiscUtils.js.map +1 -1
  61. package/dist/esm/utils/PlaywrightUtils.d.ts +0 -6
  62. package/dist/esm/utils/PlaywrightUtils.d.ts.map +1 -1
  63. package/dist/esm/utils/PlaywrightUtils.js +12 -7
  64. package/dist/esm/utils/PlaywrightUtils.js.map +1 -1
  65. package/dist/lib/ai/PageAi.d.ts.map +1 -1
  66. package/dist/lib/ai/PageAi.js +3 -1
  67. package/dist/lib/ai/PageAi.js.map +1 -1
  68. package/dist/lib/page/DonobuExtendedPage.d.ts +35 -0
  69. package/dist/lib/page/DonobuExtendedPage.d.ts.map +1 -1
  70. package/dist/lib/page/extendPage.d.ts +2 -0
  71. package/dist/lib/page/extendPage.d.ts.map +1 -1
  72. package/dist/lib/page/extendPage.js +7 -0
  73. package/dist/lib/page/extendPage.js.map +1 -1
  74. package/dist/lib/page/index.d.ts +1 -0
  75. package/dist/lib/page/index.d.ts.map +1 -1
  76. package/dist/lib/page/index.js +3 -0
  77. package/dist/lib/page/index.js.map +1 -1
  78. package/dist/lib/page/tbd/actionRecorder.d.ts +31 -0
  79. package/dist/lib/page/tbd/actionRecorder.d.ts.map +1 -0
  80. package/dist/lib/page/tbd/actionRecorder.js +85 -0
  81. package/dist/lib/page/tbd/actionRecorder.js.map +1 -0
  82. package/dist/lib/page/tbd/callSiteCapture.d.ts +21 -0
  83. package/dist/lib/page/tbd/callSiteCapture.d.ts.map +1 -0
  84. package/dist/lib/page/tbd/callSiteCapture.js +85 -0
  85. package/dist/lib/page/tbd/callSiteCapture.js.map +1 -0
  86. package/dist/lib/page/tbd/fileRewriter.d.ts +10 -0
  87. package/dist/lib/page/tbd/fileRewriter.d.ts.map +1 -0
  88. package/dist/lib/page/tbd/fileRewriter.js +174 -0
  89. package/dist/lib/page/tbd/fileRewriter.js.map +1 -0
  90. package/dist/lib/page/tbd/tbdTypes.d.ts +23 -0
  91. package/dist/lib/page/tbd/tbdTypes.d.ts.map +1 -0
  92. package/dist/lib/page/tbd/tbdTypes.js +3 -0
  93. package/dist/lib/page/tbd/tbdTypes.js.map +1 -0
  94. package/dist/lib/page/tbd.d.ts +19 -0
  95. package/dist/lib/page/tbd.d.ts.map +1 -0
  96. package/dist/lib/page/tbd.js +378 -0
  97. package/dist/lib/page/tbd.js.map +1 -0
  98. package/dist/lib/test/testExtension.d.ts.map +1 -1
  99. package/dist/lib/test/testExtension.js +15 -0
  100. package/dist/lib/test/testExtension.js.map +1 -1
  101. package/dist/managers/EnvDataManager.d.ts +13 -1
  102. package/dist/managers/EnvDataManager.d.ts.map +1 -1
  103. package/dist/managers/EnvDataManager.js +13 -1
  104. package/dist/managers/EnvDataManager.js.map +1 -1
  105. package/dist/persistence/env/EnvPersistence.d.ts +20 -0
  106. package/dist/persistence/env/EnvPersistence.d.ts.map +1 -1
  107. package/dist/persistence/env/EnvPersistenceRegistry.d.ts +9 -1
  108. package/dist/persistence/env/EnvPersistenceRegistry.d.ts.map +1 -1
  109. package/dist/persistence/env/EnvPersistenceRegistry.js +1 -1
  110. package/dist/persistence/env/EnvPersistenceRegistry.js.map +1 -1
  111. package/dist/persistence/env/EnvPersistenceSqlite.d.ts +5 -1
  112. package/dist/persistence/env/EnvPersistenceSqlite.d.ts.map +1 -1
  113. package/dist/persistence/env/EnvPersistenceSqlite.js +5 -1
  114. package/dist/persistence/env/EnvPersistenceSqlite.js.map +1 -1
  115. package/dist/persistence/env/EnvPersistenceVolatile.d.ts +8 -1
  116. package/dist/persistence/env/EnvPersistenceVolatile.d.ts.map +1 -1
  117. package/dist/persistence/env/EnvPersistenceVolatile.js +8 -1
  118. package/dist/persistence/env/EnvPersistenceVolatile.js.map +1 -1
  119. package/dist/utils/MiscUtils.d.ts +5 -0
  120. package/dist/utils/MiscUtils.d.ts.map +1 -1
  121. package/dist/utils/MiscUtils.js +12 -5
  122. package/dist/utils/MiscUtils.js.map +1 -1
  123. package/dist/utils/PlaywrightUtils.d.ts +0 -6
  124. package/dist/utils/PlaywrightUtils.d.ts.map +1 -1
  125. package/dist/utils/PlaywrightUtils.js +12 -7
  126. package/dist/utils/PlaywrightUtils.js.map +1 -1
  127. package/package.json +1 -1
@@ -0,0 +1,530 @@
1
+ <!doctype html>
2
+ <html>
3
+ <head>
4
+ <meta charset="utf-8" />
5
+ <title>Donobu — Interactive Session</title>
6
+ <style>
7
+ :root {
8
+ font-family:
9
+ -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial,
10
+ sans-serif;
11
+ font-size: 13px;
12
+ --accent: #fbbc05;
13
+ --panel-bg: #1a1a1a;
14
+ }
15
+ * {
16
+ box-sizing: border-box;
17
+ margin: 0;
18
+ padding: 0;
19
+ }
20
+ html,
21
+ body {
22
+ margin: 0;
23
+ height: 100%;
24
+ background: var(--panel-bg);
25
+ color: #fff;
26
+ overflow: hidden;
27
+ }
28
+ body {
29
+ display: flex;
30
+ flex-direction: column;
31
+ }
32
+
33
+ /* Toolbar --------------------------------------------------------- */
34
+ #toolbar {
35
+ display: flex;
36
+ align-items: center;
37
+ padding: 8px 12px;
38
+ background: rgba(255, 255, 255, 0.03);
39
+ border-bottom: 1px solid rgba(255, 255, 255, 0.08);
40
+ flex-shrink: 0;
41
+ gap: 6px;
42
+ }
43
+ #headline {
44
+ font-weight: 500;
45
+ font-size: 13px;
46
+ letter-spacing: -0.2px;
47
+ flex: 1;
48
+ white-space: nowrap;
49
+ overflow: hidden;
50
+ text-overflow: ellipsis;
51
+ }
52
+
53
+ .icon-btn {
54
+ display: none;
55
+ cursor: pointer;
56
+ opacity: 0.4;
57
+ transition:
58
+ opacity 0.2s ease,
59
+ transform 0.1s ease,
60
+ background-color 0.2s ease;
61
+ border-radius: 6px;
62
+ padding: 4px;
63
+ background: transparent;
64
+ flex-shrink: 0;
65
+ }
66
+ .icon-btn:hover {
67
+ opacity: 0.7;
68
+ background: rgba(255, 255, 255, 0.1);
69
+ }
70
+ .icon-btn.visible {
71
+ display: flex;
72
+ opacity: 1;
73
+ background: rgba(251, 188, 5, 0.15);
74
+ }
75
+ .icon-btn.visible:hover {
76
+ background: rgba(251, 188, 5, 0.25);
77
+ }
78
+ .icon-btn:active {
79
+ transform: scale(0.9);
80
+ transition: transform 0.05s;
81
+ }
82
+ .icon-btn svg {
83
+ width: 20px;
84
+ height: 20px;
85
+ fill: none;
86
+ stroke: currentColor;
87
+ stroke-width: 2;
88
+ stroke-linecap: round;
89
+ stroke-linejoin: round;
90
+ }
91
+
92
+ .pending {
93
+ color: var(--accent);
94
+ animation: pulse 1.5s ease-in-out infinite;
95
+ }
96
+ @keyframes pulse {
97
+ 0%,
98
+ 100% {
99
+ opacity: 1;
100
+ }
101
+ 50% {
102
+ opacity: 0.7;
103
+ }
104
+ }
105
+
106
+ /* Screencast viewport ---------------------------------------------- */
107
+ #viewport {
108
+ flex: 1;
109
+ position: relative;
110
+ overflow: hidden;
111
+ background: #000;
112
+ cursor: default;
113
+ }
114
+ #viewport.no-mirror {
115
+ display: flex;
116
+ align-items: center;
117
+ justify-content: center;
118
+ background: var(--panel-bg);
119
+ }
120
+ #viewport.no-mirror::after {
121
+ content: 'Browser is visible — interact directly';
122
+ color: rgba(255, 255, 255, 0.3);
123
+ font-size: 14px;
124
+ }
125
+ #screencast {
126
+ position: absolute;
127
+ top: 0;
128
+ left: 0;
129
+ width: 100%;
130
+ height: 100%;
131
+ object-fit: contain;
132
+ image-rendering: -webkit-optimize-contrast;
133
+ }
134
+
135
+ /* Instructions bar ------------------------------------------------- */
136
+ #instructions-bar {
137
+ display: flex;
138
+ align-items: center;
139
+ padding: 8px 12px;
140
+ gap: 8px;
141
+ border-top: 1px solid rgba(255, 255, 255, 0.08);
142
+ background: rgba(255, 255, 255, 0.03);
143
+ flex-shrink: 0;
144
+ }
145
+ #msgBox {
146
+ flex: 1;
147
+ min-height: 32px;
148
+ max-height: 80px;
149
+ resize: none;
150
+ background: rgba(255, 255, 255, 0.95);
151
+ color: #333;
152
+ border: 1px solid rgba(255, 255, 255, 0.2);
153
+ border-radius: 6px;
154
+ padding: 6px 10px;
155
+ font-family: inherit;
156
+ font-size: 12px;
157
+ }
158
+ #msgBox:focus {
159
+ border-color: rgba(251, 188, 5, 0.8);
160
+ box-shadow: 0 0 0 2px rgba(251, 188, 5, 0.2);
161
+ outline: none;
162
+ }
163
+ #msgBox::placeholder {
164
+ color: #999;
165
+ }
166
+ #msgBox:disabled {
167
+ background: rgba(255, 255, 255, 0.3);
168
+ color: #666;
169
+ }
170
+ #sendBtn {
171
+ display: flex;
172
+ cursor: pointer;
173
+ padding: 4px;
174
+ border-radius: 6px;
175
+ background: rgba(251, 188, 5, 0.15);
176
+ opacity: 0.8;
177
+ transition:
178
+ opacity 0.2s,
179
+ background-color 0.2s;
180
+ flex-shrink: 0;
181
+ }
182
+ #sendBtn:hover {
183
+ opacity: 1;
184
+ background: rgba(251, 188, 5, 0.3);
185
+ }
186
+ #sendBtn:active {
187
+ transform: scale(0.9);
188
+ }
189
+ #sendBtn.hidden {
190
+ display: none;
191
+ }
192
+ #sendBtn svg {
193
+ width: 20px;
194
+ height: 20px;
195
+ fill: none;
196
+ stroke: currentColor;
197
+ stroke-width: 2;
198
+ stroke-linecap: round;
199
+ stroke-linejoin: round;
200
+ }
201
+ </style>
202
+ </head>
203
+
204
+ <body>
205
+ <!-- Toolbar -->
206
+ <div id="toolbar">
207
+ <!-- Pause: shown when AI is running -->
208
+ <span id="btnPause" class="icon-btn" title="Pause AI">
209
+ <svg viewBox="0 0 24 24">
210
+ <circle cx="12" cy="12" r="10" />
211
+ <line x1="10" y1="8" x2="10" y2="16" />
212
+ <line x1="14" y1="8" x2="14" y2="16" />
213
+ </svg>
214
+ </span>
215
+ <!-- Play: shown when AI is paused (resume) -->
216
+ <span id="btnPlay" class="icon-btn" title="Resume AI">
217
+ <svg viewBox="0 0 24 24">
218
+ <circle cx="12" cy="12" r="10" />
219
+ <polygon points="10 8 16 12 10 16 10 8" />
220
+ </svg>
221
+ </span>
222
+ <span id="headline">Explore the page, or give the AI an instruction</span>
223
+ </div>
224
+
225
+ <!-- Screencast viewport (only used in headless mirror mode) -->
226
+ <div id="viewport" class="no-mirror">
227
+ <canvas id="screencast"></canvas>
228
+ </div>
229
+
230
+ <!-- Instructions bar -->
231
+ <div id="instructions-bar">
232
+ <input id="msgBox" type="text" placeholder="Tell the AI what to do…" />
233
+ <span id="sendBtn" title="Send instruction">
234
+ <svg viewBox="0 0 24 24">
235
+ <line x1="22" y1="2" x2="11" y2="13" />
236
+ <polygon points="22 2 15 22 11 13 2 9 22 2" />
237
+ </svg>
238
+ </span>
239
+ </div>
240
+
241
+ <script>
242
+ const ipc = (() => {
243
+ if (window.electron) {
244
+ return { send: window.electron.send, on: window.electron.on };
245
+ }
246
+ if (window.__controlPanelSend) {
247
+ window.__controlPanelListeners = {};
248
+ return {
249
+ send: (channel, data) => {
250
+ window.__controlPanelSend(JSON.stringify({ channel, data }));
251
+ },
252
+ on: (channel, listener) => {
253
+ window.__controlPanelListeners[channel] = listener;
254
+ },
255
+ };
256
+ }
257
+ throw new Error('IPC bridge missing');
258
+ })();
259
+
260
+ const { send, on } = ipc;
261
+ const flowId = new URLSearchParams(location.search).get('flow');
262
+ const viewport = document.getElementById('viewport');
263
+ const canvas = document.getElementById('screencast');
264
+ const ctx = canvas.getContext('2d');
265
+ const btnPause = document.getElementById('btnPause');
266
+ const btnPlay = document.getElementById('btnPlay');
267
+ const headline = document.getElementById('headline');
268
+ const msgBox = document.getElementById('msgBox');
269
+ const sendBtn = document.getElementById('sendBtn');
270
+
271
+ // --- Panel state management ----------------------------------------
272
+ let panelState = 'idle'; // 'idle' | 'running' | 'paused'
273
+
274
+ function setPanelState(newState, headlineText) {
275
+ panelState = newState;
276
+ headline.classList.remove('pending');
277
+
278
+ if (newState === 'idle') {
279
+ headline.textContent =
280
+ headlineText || 'Explore the page, or give the AI an instruction';
281
+ msgBox.disabled = false;
282
+ btnPause.classList.remove('visible');
283
+ btnPlay.classList.remove('visible');
284
+ sendBtn.classList.remove('hidden');
285
+ } else if (newState === 'running') {
286
+ headline.textContent = headlineText || 'AI is working…';
287
+ headline.classList.add('pending');
288
+ msgBox.disabled = true;
289
+ btnPause.classList.add('visible');
290
+ btnPlay.classList.remove('visible');
291
+ sendBtn.classList.add('hidden');
292
+ } else if (newState === 'paused') {
293
+ headline.textContent =
294
+ headlineText ||
295
+ 'AI paused — interact manually or revise instructions';
296
+ msgBox.disabled = false;
297
+ btnPause.classList.remove('visible');
298
+ btnPlay.classList.add('visible');
299
+ sendBtn.classList.remove('hidden');
300
+ }
301
+ }
302
+
303
+ // --- State updates from backend ------------------------------------
304
+ on('stateUpdate', (_event, data) => {
305
+ const state = data.state;
306
+ const hl = data.headline;
307
+
308
+ if (state === 'PAUSED') {
309
+ setPanelState('paused', hl);
310
+ } else if (
311
+ state === 'RUNNING_ACTION' ||
312
+ state === 'QUERYING_LLM_FOR_NEXT_ACTION' ||
313
+ state === 'INITIALIZING' ||
314
+ state === 'RESUMING'
315
+ ) {
316
+ setPanelState('running', hl);
317
+ } else if (
318
+ state === 'IDLE' ||
319
+ state === 'SUCCESS' ||
320
+ state === 'FAILED'
321
+ ) {
322
+ setPanelState('idle', hl);
323
+ }
324
+ });
325
+
326
+ // --- Screencast rendering -----------------------------------------
327
+ let screencastActive = false;
328
+ let imgScale = { x: 1, y: 1 };
329
+ let screencastOffsetX = 0;
330
+ let screencastOffsetY = 0;
331
+ let screencastWidth = 0;
332
+ let screencastHeight = 0;
333
+
334
+ on('screencastFrame', (_event, { data, metadata }) => {
335
+ if (!screencastActive) {
336
+ screencastActive = true;
337
+ viewport.classList.remove('no-mirror');
338
+ }
339
+
340
+ const img = new Image();
341
+ img.onload = () => {
342
+ const vw = viewport.clientWidth;
343
+ const vh = viewport.clientHeight;
344
+ const iw = metadata.deviceWidth;
345
+ const ih = metadata.deviceHeight;
346
+ const scale = Math.min(vw / iw, vh / ih);
347
+
348
+ screencastWidth = iw * scale;
349
+ screencastHeight = ih * scale;
350
+ screencastOffsetX = (vw - screencastWidth) / 2;
351
+ screencastOffsetY = (vh - screencastHeight) / 2;
352
+
353
+ canvas.width = vw;
354
+ canvas.height = vh;
355
+ canvas.style.width = vw + 'px';
356
+ canvas.style.height = vh + 'px';
357
+
358
+ ctx.clearRect(0, 0, vw, vh);
359
+ ctx.drawImage(
360
+ img,
361
+ screencastOffsetX,
362
+ screencastOffsetY,
363
+ screencastWidth,
364
+ screencastHeight,
365
+ );
366
+
367
+ imgScale.x = iw / screencastWidth;
368
+ imgScale.y = ih / screencastHeight;
369
+ };
370
+ img.src = 'data:image/jpeg;base64,' + data;
371
+ });
372
+
373
+ // --- Input forwarding (screencast mode only) ----------------------
374
+ function translateCoords(clientX, clientY) {
375
+ const rect = canvas.getBoundingClientRect();
376
+ const cx = clientX - rect.left - screencastOffsetX;
377
+ const cy = clientY - rect.top - screencastOffsetY;
378
+ return {
379
+ x: Math.round(cx * imgScale.x),
380
+ y: Math.round(cy * imgScale.y),
381
+ };
382
+ }
383
+
384
+ function isInBounds(clientX, clientY) {
385
+ const rect = canvas.getBoundingClientRect();
386
+ const cx = clientX - rect.left;
387
+ const cy = clientY - rect.top;
388
+ return (
389
+ cx >= screencastOffsetX &&
390
+ cx <= screencastOffsetX + screencastWidth &&
391
+ cy >= screencastOffsetY &&
392
+ cy <= screencastOffsetY + screencastHeight
393
+ );
394
+ }
395
+
396
+ viewport.addEventListener('mousedown', (e) => {
397
+ if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
398
+ const { x, y } = translateCoords(e.clientX, e.clientY);
399
+ send('cdpInput', {
400
+ type: 'mousePressed',
401
+ x,
402
+ y,
403
+ button: 'left',
404
+ clickCount: 1,
405
+ });
406
+ });
407
+
408
+ viewport.addEventListener('mouseup', (e) => {
409
+ if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
410
+ const { x, y } = translateCoords(e.clientX, e.clientY);
411
+ send('cdpInput', {
412
+ type: 'mouseReleased',
413
+ x,
414
+ y,
415
+ button: 'left',
416
+ clickCount: 1,
417
+ });
418
+ });
419
+
420
+ viewport.addEventListener('mousemove', (e) => {
421
+ if (!screencastActive || !isInBounds(e.clientX, e.clientY)) return;
422
+ const { x, y } = translateCoords(e.clientX, e.clientY);
423
+ send('cdpInput', { type: 'mouseMoved', x, y });
424
+ });
425
+
426
+ viewport.addEventListener(
427
+ 'wheel',
428
+ (e) => {
429
+ if (!screencastActive) return;
430
+ e.preventDefault();
431
+ const { x, y } = translateCoords(e.clientX, e.clientY);
432
+ send('cdpInput', {
433
+ type: 'mouseWheel',
434
+ x,
435
+ y,
436
+ deltaX: e.deltaX,
437
+ deltaY: e.deltaY,
438
+ });
439
+ },
440
+ { passive: false },
441
+ );
442
+
443
+ // Keyboard input — only forward when canvas area is focused
444
+ viewport.tabIndex = 0;
445
+ viewport.addEventListener('keydown', (e) => {
446
+ if (!screencastActive) return;
447
+ e.preventDefault();
448
+ send('cdpInput', {
449
+ type: 'keyDown',
450
+ key: e.key,
451
+ code: e.code,
452
+ modifiers: getModifiers(e),
453
+ text: e.key.length === 1 ? e.key : '',
454
+ });
455
+ });
456
+ viewport.addEventListener('keyup', (e) => {
457
+ if (!screencastActive) return;
458
+ e.preventDefault();
459
+ send('cdpInput', {
460
+ type: 'keyUp',
461
+ key: e.key,
462
+ code: e.code,
463
+ modifiers: getModifiers(e),
464
+ });
465
+ });
466
+
467
+ function getModifiers(e) {
468
+ let m = 0;
469
+ if (e.altKey) m |= 1;
470
+ if (e.ctrlKey) m |= 2;
471
+ if (e.metaKey) m |= 4;
472
+ if (e.shiftKey) m |= 8;
473
+ return m;
474
+ }
475
+
476
+ // --- User actions --------------------------------------------------
477
+
478
+ function submitInstruction() {
479
+ const text = msgBox.value.trim();
480
+ if (!text) return;
481
+ msgBox.value = '';
482
+
483
+ if (panelState === 'idle') {
484
+ // Start a new AI instruction
485
+ send('userAction', {
486
+ flowId,
487
+ action: { type: 'RESUME', userInstruction: text },
488
+ });
489
+ setPanelState('running');
490
+ } else if (panelState === 'paused') {
491
+ // Resume AI with revised instructions
492
+ send('userAction', {
493
+ flowId,
494
+ action: { type: 'RESUME', userInstruction: text },
495
+ });
496
+ setPanelState('running');
497
+ }
498
+ }
499
+
500
+ // Send button
501
+ sendBtn.onclick = submitInstruction;
502
+
503
+ // Enter to submit (only when idle or paused)
504
+ msgBox.addEventListener('keydown', (e) => {
505
+ if (e.key === 'Enter' && !e.shiftKey) {
506
+ e.preventDefault();
507
+ submitInstruction();
508
+ }
509
+ });
510
+
511
+ // Pause button — pauses the AI mid-execution
512
+ btnPause.onclick = () => {
513
+ send('userAction', {
514
+ flowId,
515
+ action: { type: 'PAUSE' },
516
+ });
517
+ setPanelState('paused');
518
+ };
519
+
520
+ // Play button — resume without new instructions
521
+ btnPlay.onclick = () => {
522
+ send('userAction', {
523
+ flowId,
524
+ action: { type: 'RESUME' },
525
+ });
526
+ setPanelState('running');
527
+ };
528
+ </script>
529
+ </body>
530
+ </html>