demo-dev 0.0.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +174 -0
  2. package/bin/demo-cli.js +26 -0
  3. package/bin/demo-dev.js +26 -0
  4. package/demo.dev.config.example.json +20 -0
  5. package/dist/index.d.ts +392 -0
  6. package/dist/index.js +2116 -0
  7. package/package.json +76 -0
  8. package/skills/demo-dev/SKILL.md +153 -0
  9. package/skills/demo-dev/references/configuration.md +102 -0
  10. package/skills/demo-dev/references/recipes.md +83 -0
  11. package/src/ai/provider.ts +254 -0
  12. package/src/auth/bootstrap.ts +72 -0
  13. package/src/browser/session.ts +43 -0
  14. package/src/capture/continuous-capture.ts +739 -0
  15. package/src/cli.ts +337 -0
  16. package/src/config/project.ts +183 -0
  17. package/src/github/comment.ts +134 -0
  18. package/src/index.ts +10 -0
  19. package/src/lib/data-uri.ts +21 -0
  20. package/src/lib/fs.ts +7 -0
  21. package/src/lib/git.ts +59 -0
  22. package/src/lib/media.ts +23 -0
  23. package/src/orchestrate.ts +166 -0
  24. package/src/planner/heuristic.ts +180 -0
  25. package/src/planner/index.ts +26 -0
  26. package/src/planner/llm.ts +85 -0
  27. package/src/planner/openai.ts +77 -0
  28. package/src/planner/prompt.ts +331 -0
  29. package/src/planner/refine.ts +155 -0
  30. package/src/planner/schema.ts +62 -0
  31. package/src/presentation/polish.ts +84 -0
  32. package/src/probe/page-probe.ts +225 -0
  33. package/src/render/browser-frame.ts +176 -0
  34. package/src/render/ffmpeg-compose.ts +779 -0
  35. package/src/render/visual-plan.ts +422 -0
  36. package/src/setup/doctor.ts +158 -0
  37. package/src/setup/init.ts +90 -0
  38. package/src/types.ts +105 -0
  39. package/src/voice/script.ts +42 -0
  40. package/src/voice/tts.ts +286 -0
  41. package/tsconfig.json +16 -0
@@ -0,0 +1,739 @@
1
+ /**
2
+ * Continuous capture module.
3
+ *
4
+ * Instead of recording each scene as a separate browser context (which produces
5
+ * choppy, slideshow-like output), this module runs ALL scenes in a single
6
+ * continuous Playwright session while:
7
+ *
8
+ * 1. Using the Playwright 1.59 `page.screencast` API for a single unbroken
9
+ * WebM recording.
10
+ * 2. Using `ghost-cursor-playwright` for human-like Bézier-curve mouse
11
+ * movement (Fitts's Law, overshoot, random landing points inside elements).
12
+ * 3. Logging high-frequency cursor positions + interaction events as metadata
13
+ * so post-processing can generate intelligent zoom keyframes à la Screen
14
+ * Studio.
15
+ *
16
+ * The entire module is headless-capable and CI-friendly.
17
+ */
18
+
19
+ import { mkdir } from "node:fs/promises";
20
+ import { join } from "node:path";
21
+ import { chromium, type Page, type Locator } from "playwright";
22
+ import { createCursor } from "ghost-cursor-playwright";
23
+ import {
24
+ getContextOptionsWithSession,
25
+ persistSessionState,
26
+ resolveSessionConfig,
27
+ } from "../browser/session.js";
28
+ import type {
29
+ ActionTarget,
30
+ DemoPlan,
31
+ SceneAction,
32
+ } from "../types.js";
33
+
34
+ // ---------------------------------------------------------------------------
35
+ // Types
36
+ // ---------------------------------------------------------------------------
37
+
38
+ /** A single logged cursor position sample. */
39
+ export interface CursorSample {
40
+ /** Milliseconds since capture start */
41
+ atMs: number;
42
+ x: number;
43
+ y: number;
44
+ }
45
+
46
+ /** A logged interaction event with screen coordinates + timing. */
47
+ export interface CaptureInteraction {
48
+ type: SceneAction["type"] | "scene-start" | "scene-end" | "stable";
49
+ sceneId: string;
50
+ atMs: number;
51
+ x?: number;
52
+ y?: number;
53
+ width?: number;
54
+ height?: number;
55
+ /** For fill actions – the value typed. */
56
+ fillValue?: string;
57
+ }
58
+
59
+ /** Scene timing marker within the continuous recording. */
60
+ export interface SceneMarker {
61
+ sceneId: string;
62
+ sceneTitle: string;
63
+ startMs: number;
64
+ endMs: number;
65
+ url: string;
66
+ }
67
+
68
+ /** Full output of a continuous capture session. */
69
+ export interface ContinuousCaptureResult {
70
+ /** Path to the single continuous WebM recording. */
71
+ videoPath: string;
72
+ /** High-frequency cursor position log. */
73
+ cursorLog: CursorSample[];
74
+ /** Interaction events (clicks, fills, navigations, etc.). */
75
+ interactions: CaptureInteraction[];
76
+ /** Per-scene timing markers. */
77
+ sceneMarkers: SceneMarker[];
78
+ /** Total duration in milliseconds. */
79
+ totalDurationMs: number;
80
+ /** Viewport size used for the recording. */
81
+ viewport: { width: number; height: number };
82
+ }
83
+
84
+ // ---------------------------------------------------------------------------
85
+ // Helpers
86
+ // ---------------------------------------------------------------------------
87
+
88
+ /** Strip trailing numbers/badges that LLMs often include (e.g. "Positive reply8" → "Positive reply"). */
89
+ const stripBadge = (text: string) => text.replace(/\d+\s*$/, "").trim();
90
+
91
+ /** Core locator resolution — exact match. */
92
+ const resolveLocatorExact = (page: Page, target: ActionTarget): Locator => {
93
+ switch (target.strategy) {
94
+ case "label":
95
+ return page.getByLabel(target.value, { exact: target.exact });
96
+ case "text":
97
+ return page.getByText(target.value, { exact: target.exact });
98
+ case "placeholder":
99
+ return page.getByPlaceholder(target.value, { exact: target.exact });
100
+ case "testId":
101
+ return page.getByTestId(target.value);
102
+ case "css":
103
+ return page.locator(target.value);
104
+ case "role":
105
+ return page.getByRole(target.role as never, {
106
+ name: target.name,
107
+ exact: target.exact,
108
+ });
109
+ }
110
+ };
111
+
112
+ /**
113
+ * Resolve a locator with fuzzy fallback.
114
+ * Tries exact match first, then progressively looser strategies:
115
+ * 1. Strip trailing numbers/badges from text
116
+ * 2. Use non-exact (substring) matching
117
+ * 3. For role targets, try text strategy instead
118
+ */
119
+ const resolveLocator = (page: Page, target: ActionTarget): Locator => {
120
+ const exact = resolveLocatorExact(page, target);
121
+
122
+ // Build a chain of fallback locators using .or()
123
+ let result = exact;
124
+
125
+ if (target.strategy === "text" || target.strategy === "label") {
126
+ const stripped = stripBadge(target.value);
127
+ if (stripped !== target.value && stripped.length > 0) {
128
+ // Try without the badge number
129
+ result = result.or(
130
+ target.strategy === "text"
131
+ ? page.getByText(stripped, { exact: false })
132
+ : page.getByLabel(stripped, { exact: false }),
133
+ );
134
+ }
135
+ if (target.exact) {
136
+ // Also try non-exact (substring) match
137
+ result = result.or(
138
+ target.strategy === "text"
139
+ ? page.getByText(target.value, { exact: false })
140
+ : page.getByLabel(target.value, { exact: false }),
141
+ );
142
+ }
143
+ }
144
+
145
+ if (target.strategy === "role" && target.name) {
146
+ const stripped = stripBadge(target.name);
147
+ if (stripped !== target.name && stripped.length > 0) {
148
+ // Try role with stripped name
149
+ result = result.or(
150
+ page.getByRole(target.role as never, { name: stripped, exact: false }),
151
+ );
152
+ }
153
+ // Also try as text match (LLMs sometimes confuse role vs text)
154
+ result = result.or(page.getByText(target.name, { exact: false }));
155
+ if (stripped !== target.name) {
156
+ result = result.or(page.getByText(stripped, { exact: false }));
157
+ }
158
+ }
159
+
160
+ return result;
161
+ };
162
+
163
+ /** Random delay that feels "human": base ± jitter. */
164
+ const humanDelay = (baseMs: number, jitter = 0.4) => {
165
+ const factor = 1 + (Math.random() * 2 - 1) * jitter;
166
+ return Math.max(50, Math.round(baseMs * factor));
167
+ };
168
+
169
+ /** Get bounding-box center for an action's target element. */
170
+ const boxForAction = async (
171
+ page: Page,
172
+ action: SceneAction,
173
+ ): Promise<{ x: number; y: number; width: number; height: number } | undefined> => {
174
+ const getBox = async (locator: Locator) => {
175
+ const box = await locator.first().boundingBox().catch(() => null);
176
+ if (!box) return undefined;
177
+ return {
178
+ x: box.x + box.width / 2,
179
+ y: box.y + box.height / 2,
180
+ width: box.width,
181
+ height: box.height,
182
+ };
183
+ };
184
+
185
+ switch (action.type) {
186
+ case "click":
187
+ case "hover":
188
+ case "fill":
189
+ case "select":
190
+ case "dragSelect":
191
+ return getBox(resolveLocator(page, action.target));
192
+ case "scrollIntoView":
193
+ return getBox(resolveLocator(page, action.target));
194
+ case "waitForText":
195
+ return getBox(page.getByText(action.value, { exact: action.exact }));
196
+ default:
197
+ return undefined;
198
+ }
199
+ };
200
+
201
+ // ---------------------------------------------------------------------------
202
+ // Cursor position tracking
203
+ // ---------------------------------------------------------------------------
204
+
205
+ /**
206
+ * Inject a JS snippet into the page that tracks mouse position at ~60 fps
207
+ * and stores samples on `window.__cursorSamples`. We poll these from Node.
208
+ */
209
+ const CURSOR_TRACKER_SCRIPT = `
210
+ (() => {
211
+ if (window.__cursorTrackerInstalled) return;
212
+ window.__cursorTrackerInstalled = true;
213
+ window.__cursorSamples = [];
214
+ window.__cursorTrackingStart = Date.now();
215
+ let lastX = 0, lastY = 0;
216
+
217
+ document.addEventListener('mousemove', (e) => {
218
+ lastX = e.clientX;
219
+ lastY = e.clientY;
220
+ }, { passive: true });
221
+
222
+ const tick = () => {
223
+ window.__cursorSamples.push({
224
+ atMs: Date.now() - window.__cursorTrackingStart,
225
+ x: lastX,
226
+ y: lastY,
227
+ });
228
+ requestAnimationFrame(tick);
229
+ };
230
+ requestAnimationFrame(tick);
231
+ })();
232
+ `;
233
+
234
+ // ---------------------------------------------------------------------------
235
+ // CSS-based zoom (Screen Studio style — animated in browser at 60fps)
236
+ // ---------------------------------------------------------------------------
237
+
238
+ const ZOOM_INJECT_SCRIPT = `
239
+ (() => {
240
+ if (window.__zoomInstalled) return;
241
+ window.__zoomInstalled = true;
242
+ const html = document.documentElement;
243
+ html.style.transition = 'transform 0.8s cubic-bezier(0.22, 0.61, 0.36, 1)';
244
+ html.style.transformOrigin = '0 0';
245
+ window.__zoomTo = (x, y, scale) => {
246
+ const vw = window.innerWidth;
247
+ const vh = window.innerHeight;
248
+ const tx = -(x - vw / scale / 2) * (scale - 1);
249
+ const ty = -(y - vh / scale / 2) * (scale - 1);
250
+ const clampedTx = Math.min(0, Math.max(tx, -(vw * (scale - 1))));
251
+ const clampedTy = Math.min(0, Math.max(ty, -(vh * (scale - 1))));
252
+ html.style.transform = 'scale(' + scale + ') translate(' + clampedTx / scale + 'px, ' + clampedTy / scale + 'px)';
253
+ };
254
+ window.__zoomReset = () => {
255
+ html.style.transform = 'scale(1) translate(0px, 0px)';
256
+ };
257
+ })();
258
+ `;
259
+
260
+ /** Smoothly zoom into a point on the page (captured by screencast at 60fps). */
261
+ const zoomToPoint = async (page: Page, x: number, y: number, scale = 1.5) => {
262
+ await page.evaluate(ZOOM_INJECT_SCRIPT).catch(() => undefined);
263
+ await page.evaluate(
264
+ ({ x, y, scale }) => (window as any).__zoomTo?.(x, y, scale),
265
+ { x, y, scale },
266
+ ).catch(() => undefined);
267
+ // Wait for CSS transition to complete (0.8s)
268
+ await page.waitForTimeout(850);
269
+ };
270
+
271
+ /** Smoothly zoom back to 1x. */
272
+ const zoomReset = async (page: Page) => {
273
+ await page.evaluate(() => (window as any).__zoomReset?.()).catch(() => undefined);
274
+ await page.waitForTimeout(850);
275
+ };
276
+
277
+ /** Drain cursor samples from the page and append to our log. */
278
+ const drainCursorSamples = async (
279
+ page: Page,
280
+ log: CursorSample[],
281
+ offsetMs: number,
282
+ ): Promise<void> => {
283
+ try {
284
+ const samples: CursorSample[] = await page.evaluate(() => {
285
+ const s = (window as any).__cursorSamples ?? [];
286
+ (window as any).__cursorSamples = [];
287
+ return s;
288
+ });
289
+ for (const s of samples) {
290
+ log.push({ atMs: s.atMs + offsetMs, x: s.x, y: s.y });
291
+ }
292
+ } catch {
293
+ // page may have navigated – silently skip
294
+ }
295
+ };
296
+
297
+ // ---------------------------------------------------------------------------
298
+ // Visible cursor rendering (for headless)
299
+ // ---------------------------------------------------------------------------
300
+
301
+ const CURSOR_OVERLAY_SCRIPT = `
302
+ (() => {
303
+ if (document.getElementById('__ghost-cursor')) return;
304
+
305
+ /* ── macOS-style pointer cursor ── */
306
+ const cursor = document.createElement('div');
307
+ cursor.id = '__ghost-cursor';
308
+ Object.assign(cursor.style, {
309
+ position: 'fixed', zIndex: '999999', pointerEvents: 'none',
310
+ left: '0px', top: '0px', width: '22px', height: '32px',
311
+ transition: 'left 0.04s cubic-bezier(.2,.8,.3,1), top 0.04s cubic-bezier(.2,.8,.3,1)',
312
+ filter: 'drop-shadow(0 2px 3px rgba(0,0,0,0.4))',
313
+ });
314
+ /* Standard macOS cursor: white fill, black outline, clean geometry */
315
+ cursor.innerHTML = '<svg width="22" height="32" viewBox="0 0 22 32" xmlns="http://www.w3.org/2000/svg">' +
316
+ '<path d="M1.5 0.5L1.5 24.5L7 18.5L11.5 28.5L14.5 27L10 17.5L18 17.5Z" fill="white" stroke="black" stroke-width="1.2" stroke-linejoin="round"/>' +
317
+ '</svg>';
318
+ document.body.appendChild(cursor);
319
+
320
+ /* ── Click ripple ── */
321
+ const ripple = document.createElement('div');
322
+ ripple.id = '__ghost-ripple';
323
+ Object.assign(ripple.style, {
324
+ position: 'fixed', zIndex: '999998', pointerEvents: 'none',
325
+ width: '40px', height: '40px', borderRadius: '50%',
326
+ border: '2px solid rgba(59,130,246,0.6)',
327
+ background: 'rgba(59,130,246,0.12)',
328
+ transform: 'translate(-50%,-50%) scale(0)',
329
+ opacity: '0', left: '0px', top: '0px',
330
+ transition: 'transform 0.35s cubic-bezier(.2,.8,.3,1), opacity 0.35s ease-out',
331
+ });
332
+ document.body.appendChild(ripple);
333
+
334
+ document.addEventListener('mousemove', (e) => {
335
+ cursor.style.left = e.clientX + 'px';
336
+ cursor.style.top = e.clientY + 'px';
337
+ }, { passive: true });
338
+
339
+ document.addEventListener('mousedown', (e) => {
340
+ /* Pointer press animation */
341
+ cursor.style.transform = 'scale(0.82)';
342
+ setTimeout(() => { cursor.style.transform = 'scale(1)'; }, 150);
343
+
344
+ /* Ripple at click position */
345
+ ripple.style.left = e.clientX + 'px';
346
+ ripple.style.top = e.clientY + 'px';
347
+ ripple.style.transform = 'translate(-50%,-50%) scale(0)';
348
+ ripple.style.opacity = '1';
349
+ /* Force reflow so transition restarts */
350
+ void ripple.offsetWidth;
351
+ ripple.style.transform = 'translate(-50%,-50%) scale(1)';
352
+ ripple.style.opacity = '0';
353
+ });
354
+ })();
355
+ `;
356
+
357
+ // ---------------------------------------------------------------------------
358
+ // Human-like action execution
359
+ // ---------------------------------------------------------------------------
360
+
361
+ type GhostCursor = Awaited<ReturnType<typeof createCursor>>;
362
+
363
+ const humanTypeInto = async (page: Page, locator: Locator, value: string) => {
364
+ await locator.first().click();
365
+ // Clear existing content
366
+ await locator.first().fill("");
367
+ // Type character by character with human-like delays
368
+ for (const char of value) {
369
+ await page.keyboard.type(char, { delay: humanDelay(65, 0.5) });
370
+ }
371
+ };
372
+
373
+ const runActionWithCursor = async (
374
+ page: Page,
375
+ action: SceneAction,
376
+ baseUrl: string,
377
+ cursor: GhostCursor,
378
+ ) => {
379
+ switch (action.type) {
380
+ case "navigate": {
381
+ const url = new URL(action.url, baseUrl).toString();
382
+ try {
383
+ await page.goto(url, { waitUntil: "networkidle", timeout: 30000 });
384
+ } catch (error) {
385
+ const message = error instanceof Error ? error.message : String(error);
386
+ if (!message.includes("page.goto: Timeout")) throw error;
387
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
388
+ await page.waitForTimeout(1500);
389
+ }
390
+ // Re-inject scripts after navigation
391
+ await page.evaluate(CURSOR_OVERLAY_SCRIPT).catch(() => undefined);
392
+ await page.evaluate(CURSOR_TRACKER_SCRIPT).catch(() => undefined);
393
+ await page.evaluate(ZOOM_INJECT_SCRIPT).catch(() => undefined);
394
+ await page.waitForTimeout(humanDelay(400));
395
+ break;
396
+ }
397
+
398
+ case "wait": {
399
+ await page.waitForTimeout(action.ms);
400
+ break;
401
+ }
402
+
403
+ case "scroll": {
404
+ // Smooth scroll instead of instant
405
+ await page.evaluate(
406
+ (y) => window.scrollBy({ top: y, behavior: "smooth" }),
407
+ action.y,
408
+ );
409
+ await page.waitForTimeout(humanDelay(500));
410
+ break;
411
+ }
412
+
413
+ case "scrollIntoView": {
414
+ const locator = resolveLocator(page, action.target);
415
+ await locator.first().scrollIntoViewIfNeeded();
416
+ await page.waitForTimeout(humanDelay(300));
417
+ break;
418
+ }
419
+
420
+ case "click": {
421
+ const selector = buildCssSelector(action.target);
422
+ if (selector) {
423
+ await cursor.actions.click({
424
+ target: selector,
425
+ waitBeforeClick: [100, 300],
426
+ });
427
+ } else {
428
+ const box = await resolveLocator(page, action.target)
429
+ .first().boundingBox().catch(() => null);
430
+ if (box) {
431
+ await cursor.actions.move({
432
+ x: box.x + box.width / 2,
433
+ y: box.y + box.height / 2,
434
+ });
435
+ await page.waitForTimeout(humanDelay(120));
436
+ await page.mouse.click(
437
+ box.x + box.width / 2,
438
+ box.y + box.height / 2,
439
+ );
440
+ } else {
441
+ await resolveLocator(page, action.target).first().click();
442
+ }
443
+ }
444
+ // Zoom in AFTER clicking to show the result (Screen Studio style)
445
+ const clickBox = await resolveLocator(page, action.target)
446
+ .first().boundingBox().catch(() => null);
447
+ if (clickBox) {
448
+ await zoomToPoint(page, clickBox.x + clickBox.width / 2, clickBox.y + clickBox.height / 2, 1.4);
449
+ await page.waitForTimeout(humanDelay(1200));
450
+ await zoomReset(page);
451
+ } else {
452
+ await page.waitForTimeout(humanDelay(200));
453
+ }
454
+ break;
455
+ }
456
+
457
+ case "hover": {
458
+ const selector = buildCssSelector(action.target);
459
+ if (selector) {
460
+ await cursor.actions.move(selector);
461
+ } else {
462
+ const box = await resolveLocator(page, action.target)
463
+ .first().boundingBox().catch(() => null);
464
+ if (box) {
465
+ await cursor.actions.move({
466
+ x: box.x + box.width / 2,
467
+ y: box.y + box.height / 2,
468
+ });
469
+ } else {
470
+ await resolveLocator(page, action.target).first().hover();
471
+ }
472
+ }
473
+ // Zoom in to emphasize what we're hovering over
474
+ const hoverBox = await resolveLocator(page, action.target)
475
+ .first().boundingBox().catch(() => null);
476
+ if (hoverBox) {
477
+ await zoomToPoint(page, hoverBox.x + hoverBox.width / 2, hoverBox.y + hoverBox.height / 2, 1.3);
478
+ await page.waitForTimeout(humanDelay(1000));
479
+ await zoomReset(page);
480
+ } else {
481
+ await page.waitForTimeout(humanDelay(300));
482
+ }
483
+ break;
484
+ }
485
+
486
+ case "fill": {
487
+ const locator = resolveLocator(page, action.target);
488
+ // Move cursor to the input first
489
+ const selector = buildCssSelector(action.target);
490
+ if (selector) {
491
+ await cursor.actions.click({
492
+ target: selector,
493
+ waitBeforeClick: [80, 200],
494
+ });
495
+ } else {
496
+ const box = await locator.first().boundingBox().catch(() => null);
497
+ if (box) {
498
+ await cursor.actions.move({
499
+ x: box.x + box.width / 2,
500
+ y: box.y + box.height / 2,
501
+ });
502
+ await page.waitForTimeout(humanDelay(100));
503
+ await page.mouse.click(
504
+ box.x + box.width / 2,
505
+ box.y + box.height / 2,
506
+ );
507
+ }
508
+ }
509
+ await page.waitForTimeout(humanDelay(150));
510
+ await humanTypeInto(page, locator, action.value);
511
+ await page.waitForTimeout(humanDelay(200));
512
+ break;
513
+ }
514
+
515
+ case "press": {
516
+ await page.keyboard.press(action.key);
517
+ await page.waitForTimeout(humanDelay(150));
518
+ break;
519
+ }
520
+
521
+ case "select": {
522
+ const locator = resolveLocator(page, action.target);
523
+ await locator.first().selectOption(action.value);
524
+ await page.waitForTimeout(humanDelay(200));
525
+ break;
526
+ }
527
+
528
+ case "dragSelect": {
529
+ const box = await resolveLocator(page, action.target)
530
+ .first()
531
+ .boundingBox();
532
+ if (!box) throw new Error("dragSelect: no bounding box");
533
+
534
+ const startX = box.x + box.width * (action.startX ?? 0.08);
535
+ const startY = box.y + box.height * (action.startY ?? 0.12);
536
+ const endX = box.x + box.width * (action.endX ?? 0.7);
537
+ const endY =
538
+ box.y + box.height * (action.endY ?? action.startY ?? 0.12);
539
+
540
+ await cursor.actions.move({ x: startX, y: startY });
541
+ await page.waitForTimeout(humanDelay(100));
542
+ await page.mouse.down();
543
+ await page.mouse.move(endX, endY, { steps: 24 });
544
+ await page.mouse.up();
545
+ await page.waitForTimeout(humanDelay(200));
546
+ break;
547
+ }
548
+
549
+ case "waitForText": {
550
+ await page
551
+ .getByText(action.value, { exact: action.exact })
552
+ .first()
553
+ .waitFor({ timeout: action.timeoutMs ?? 10000 })
554
+ .catch(() => console.warn(`[capture] waitForText timed out: "${action.value}"`));
555
+ break;
556
+ }
557
+
558
+ case "waitForUrl": {
559
+ const urlMatcher =
560
+ action.value.startsWith("http://") ||
561
+ action.value.startsWith("https://") ||
562
+ action.value.includes("*")
563
+ ? action.value
564
+ : new URL(action.value, baseUrl).toString();
565
+ await page.waitForURL(urlMatcher, { timeout: action.timeoutMs ?? 10000 })
566
+ .catch(() => console.warn(`[capture] waitForUrl timed out: "${action.value}"`));
567
+ break;
568
+ }
569
+ }
570
+ };
571
+
572
+ /** Try to produce a pure CSS selector that ghost-cursor can use.
573
+ * Playwright-specific pseudo-selectors like :has-text() are NOT valid CSS
574
+ * and will crash ghost-cursor's querySelector, so we reject them. */
575
+ const buildCssSelector = (target: ActionTarget): string | undefined => {
576
+ switch (target.strategy) {
577
+ case "css": {
578
+ // Reject Playwright-specific pseudo-selectors
579
+ if (/:has-text|:text|:visible|>>/.test(target.value)) return undefined;
580
+ return target.value;
581
+ }
582
+ case "testId":
583
+ return `[data-testid="${target.value}"]`;
584
+ default:
585
+ return undefined;
586
+ }
587
+ };
588
+
589
+ // ---------------------------------------------------------------------------
590
+ // Main entry point
591
+ // ---------------------------------------------------------------------------
592
+
593
+ export interface ContinuousCaptureOptions {
594
+ baseUrl: string;
595
+ outputDir: string;
596
+ viewport?: { width: number; height: number };
597
+ }
598
+
599
+ const DEFAULT_VIEWPORT = { width: 1600, height: 900 };
600
+
601
+ export const capturePlanContinuous = async (
602
+ plan: DemoPlan,
603
+ options: ContinuousCaptureOptions,
604
+ ): Promise<ContinuousCaptureResult> => {
605
+ const viewport = options.viewport ?? plan.scenes[0]?.viewport ?? DEFAULT_VIEWPORT;
606
+ const captureDir = join(options.outputDir, "continuous");
607
+ await mkdir(captureDir, { recursive: true });
608
+
609
+ const videoPath = join(captureDir, "recording.webm");
610
+ const session = resolveSessionConfig(options.outputDir);
611
+
612
+ const browser = await chromium.launch({ headless: true });
613
+ const context = await browser.newContext(
614
+ await getContextOptionsWithSession({ viewport }, session),
615
+ );
616
+ const page = await context.newPage();
617
+ page.setDefaultTimeout(8000);
618
+
619
+ const cursorLog: CursorSample[] = [];
620
+ const interactions: CaptureInteraction[] = [];
621
+ const sceneMarkers: SceneMarker[] = [];
622
+
623
+ const captureStartedAt = Date.now();
624
+ const elapsed = () => Date.now() - captureStartedAt;
625
+
626
+ // Start screencast recording
627
+ await page.screencast.start({ path: videoPath });
628
+
629
+ // Initialize ghost cursor
630
+ const cursor = await createCursor(page, {
631
+ overshootSpread: 2,
632
+ overshootRadius: 8,
633
+ });
634
+
635
+ // Set up a periodic drain for cursor samples
636
+ let cursorTrackingOffset = 0;
637
+ const drainInterval = setInterval(async () => {
638
+ await drainCursorSamples(page, cursorLog, cursorTrackingOffset);
639
+ }, 200);
640
+
641
+ try {
642
+ for (const scene of plan.scenes) {
643
+ const sceneStart = elapsed();
644
+
645
+ // Mark scene start
646
+ interactions.push({
647
+ type: "scene-start",
648
+ sceneId: scene.id,
649
+ atMs: sceneStart,
650
+ });
651
+
652
+ // Show chapter title via screencast overlay
653
+ await page.screencast.showChapter(scene.title).catch(() => undefined);
654
+ await page.waitForTimeout(humanDelay(600));
655
+
656
+ // Run all actions in this scene
657
+ for (const action of scene.actions) {
658
+ // Log interaction before
659
+ const boxBefore = await boxForAction(page, action).catch(() => undefined);
660
+
661
+ try {
662
+ await runActionWithCursor(page, action, options.baseUrl, cursor);
663
+ } catch (error) {
664
+ const msg = error instanceof Error ? error.message : String(error);
665
+ console.warn(`[capture] action ${action.type} failed (skipping): ${msg.split("\n")[0]}`);
666
+ }
667
+
668
+ // Re-inject scripts after navigation
669
+ if (action.type === "navigate") {
670
+ cursorTrackingOffset = elapsed();
671
+ await page.evaluate(CURSOR_TRACKER_SCRIPT).catch(() => undefined);
672
+ await page.evaluate(CURSOR_OVERLAY_SCRIPT).catch(() => undefined);
673
+ }
674
+
675
+ // Log interaction after
676
+ const boxAfter = await boxForAction(page, action);
677
+ const box = boxAfter ?? boxBefore;
678
+ interactions.push({
679
+ type: action.type,
680
+ sceneId: scene.id,
681
+ atMs: elapsed(),
682
+ x: box?.x,
683
+ y: box?.y,
684
+ width: box?.width,
685
+ height: box?.height,
686
+ fillValue: action.type === "fill" ? action.value : undefined,
687
+ });
688
+ }
689
+
690
+ // Wait for page to stabilize
691
+ await page.waitForLoadState("networkidle").catch(() => undefined);
692
+ await page.waitForTimeout(humanDelay(500));
693
+
694
+ // Persist session state
695
+ await persistSessionState(page, session);
696
+
697
+ const sceneEnd = elapsed();
698
+
699
+ // Mark scene end
700
+ interactions.push({
701
+ type: "scene-end",
702
+ sceneId: scene.id,
703
+ atMs: sceneEnd,
704
+ });
705
+
706
+ sceneMarkers.push({
707
+ sceneId: scene.id,
708
+ sceneTitle: scene.title,
709
+ startMs: sceneStart,
710
+ endMs: sceneEnd,
711
+ url: page.url(),
712
+ });
713
+
714
+ // Brief pause between scenes (feels like a human taking a breath)
715
+ await page.waitForTimeout(humanDelay(400));
716
+ }
717
+
718
+ // Final drain of cursor samples
719
+ await drainCursorSamples(page, cursorLog, cursorTrackingOffset);
720
+
721
+ // Stop recording
722
+ const totalDurationMs = elapsed();
723
+ await page.screencast.stop();
724
+ await context.close();
725
+ await browser.close();
726
+
727
+ return {
728
+ videoPath,
729
+ cursorLog,
730
+ interactions,
731
+ sceneMarkers,
732
+ totalDurationMs,
733
+ viewport,
734
+ };
735
+ } finally {
736
+ clearInterval(drainInterval);
737
+ await browser.close().catch(() => undefined);
738
+ }
739
+ };