autokap 1.5.1 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-capture.js +2 -0
- package/dist/cli-contract.d.ts +13 -0
- package/dist/cli-runner.js +21 -2
- package/dist/execution-types.d.ts +49 -9
- package/dist/mouse-animation.d.ts +6 -0
- package/dist/mouse-animation.js +8 -0
- package/dist/opcode-actions.d.ts +14 -0
- package/dist/opcode-actions.js +30 -12
- package/dist/opcode-runner.js +15 -1
- package/dist/playwright-installer.d.ts +16 -0
- package/dist/playwright-installer.js +80 -0
- package/dist/web-playwright-local.d.ts +15 -5
- package/dist/web-playwright-local.js +22 -9
- package/package.json +1 -1
package/dist/auth-capture.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { chromium } from 'playwright';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
|
+
import { ensureChromiumInstalled } from './playwright-installer.js';
|
|
3
4
|
/**
|
|
4
5
|
* Opens a headed Chromium window, lets the user log in, and uploads the
|
|
5
6
|
* resulting storageState (cookies + localStorage, HttpOnly included) to the
|
|
@@ -11,6 +12,7 @@ import { logger } from './logger.js';
|
|
|
11
12
|
*/
|
|
12
13
|
export async function captureAuthSession(options) {
|
|
13
14
|
const { apiBaseUrl, apiKey, projectId, accountId, startUrl } = options;
|
|
15
|
+
await ensureChromiumInstalled();
|
|
14
16
|
logger.info('[auth] Launching Chromium…');
|
|
15
17
|
const browser = await chromium.launch({ headless: false });
|
|
16
18
|
let context = null;
|
package/dist/cli-contract.d.ts
CHANGED
|
@@ -52,6 +52,19 @@ export interface VideoClipMetadata {
|
|
|
52
52
|
width: number;
|
|
53
53
|
height: number;
|
|
54
54
|
} | null;
|
|
55
|
+
/**
|
|
56
|
+
* For TYPE opcodes captured in clipCursor mode: clip-relative ms timestamp
|
|
57
|
+
* of every keystroke produced by `humanType`. Drives per-keystroke keyboard
|
|
58
|
+
* SFX in the video compositor.
|
|
59
|
+
*/
|
|
60
|
+
keystrokeOffsetsMs?: number[];
|
|
61
|
+
/**
|
|
62
|
+
* For CLICK / DOUBLE_CLICK / CHECK opcodes captured in clipCursor mode:
|
|
63
|
+
* clip-relative ms timestamp of each actual click dispatched by Playwright
|
|
64
|
+
* (measured AFTER the cursor animation settled). Drives mouse SFX in
|
|
65
|
+
* sync with the visible click.
|
|
66
|
+
*/
|
|
67
|
+
clickOffsetsMs?: number[];
|
|
55
68
|
}>;
|
|
56
69
|
}
|
|
57
70
|
export interface VideoAudioAsset {
|
package/dist/cli-runner.js
CHANGED
|
@@ -19,6 +19,7 @@ import { Browser } from './browser.js';
|
|
|
19
19
|
import { API_BASE_URL_ENV_VAR, requireConfig } from './cli-config.js';
|
|
20
20
|
import { WebPlaywrightLocal } from './web-playwright-local.js';
|
|
21
21
|
import { executeProgram } from './opcode-runner.js';
|
|
22
|
+
import { ensureChromiumInstalled } from './playwright-installer.js';
|
|
22
23
|
import { RecoveryChainImpl } from './recovery-chain.js';
|
|
23
24
|
import { parseProgram } from './execution-schema.js';
|
|
24
25
|
import { buildCursorOverlayScript } from './cursor-overlay-script.js';
|
|
@@ -108,6 +109,18 @@ function normalizeNumericScale(value) {
|
|
|
108
109
|
const HEALER_SYSTEM_PROMPT = 'You repair failed deterministic browser opcodes. Respond only with JSON.';
|
|
109
110
|
// ── Main entry point ────────────────────────────────────────────────
|
|
110
111
|
export async function runCapture(options) {
|
|
112
|
+
// Self-heal a missing Playwright Chromium binary BEFORE anything else.
|
|
113
|
+
// Skipped postinstalls or downstream Playwright version bumps would
|
|
114
|
+
// otherwise surface as a cryptic launch failure mid-capture.
|
|
115
|
+
try {
|
|
116
|
+
await ensureChromiumInstalled();
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
return {
|
|
120
|
+
success: false,
|
|
121
|
+
error: `playwright chromium install failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
111
124
|
const config = await requireConfig();
|
|
112
125
|
// Step 1: Get the compiled program
|
|
113
126
|
let resolvedProgram;
|
|
@@ -748,6 +761,12 @@ export function buildVideoClipMetadata(videoId, result, program, runId) {
|
|
|
748
761
|
timecodeStartMs: t.timecodeStartMs,
|
|
749
762
|
timecodeEndMs: t.timecodeEndMs,
|
|
750
763
|
bbox: t.bbox ?? null,
|
|
764
|
+
...(t.keystrokeOffsetsMs && t.keystrokeOffsetsMs.length > 0
|
|
765
|
+
? { keystrokeOffsetsMs: t.keystrokeOffsetsMs }
|
|
766
|
+
: {}),
|
|
767
|
+
...(t.clickOffsetsMs && t.clickOffsetsMs.length > 0
|
|
768
|
+
? { clickOffsetsMs: t.clickOffsetsMs }
|
|
769
|
+
: {}),
|
|
751
770
|
}));
|
|
752
771
|
clipsByKey.set(`${variantId}:${artifact.clipId}`, {
|
|
753
772
|
variantId,
|
|
@@ -951,8 +970,8 @@ async function prepareDirectArtifactUpload(params) {
|
|
|
951
970
|
clipName: artifact.clipName ?? null,
|
|
952
971
|
stepDescription: artifact.stepDescription ?? null,
|
|
953
972
|
stepIndex: typeof artifact.stepIndex === 'number' ? artifact.stepIndex : null,
|
|
954
|
-
durationMs: typeof artifact.durationMs === 'number' ? artifact.durationMs : null,
|
|
955
|
-
trimStartMs: typeof artifact.trimStartMs === 'number' ? artifact.trimStartMs : null,
|
|
973
|
+
durationMs: typeof artifact.durationMs === 'number' ? Math.round(artifact.durationMs) : null,
|
|
974
|
+
trimStartMs: typeof artifact.trimStartMs === 'number' ? Math.round(artifact.trimStartMs) : null,
|
|
956
975
|
artifactPlan: program.artifactPlan,
|
|
957
976
|
tabIconMimeType: artifact.tabIconData ? (artifact.tabIconMimeType ?? 'image/png') : null,
|
|
958
977
|
tabIconSha256,
|
|
@@ -693,6 +693,21 @@ export interface OpcodeTiming {
|
|
|
693
693
|
width: number;
|
|
694
694
|
height: number;
|
|
695
695
|
} | null;
|
|
696
|
+
/**
|
|
697
|
+
* For TYPE opcodes captured in clipCursor mode: timestamp (ms relative to the
|
|
698
|
+
* active clip start) of each individual keystroke produced by `humanType`.
|
|
699
|
+
* Drives keyboard SFX per-keystroke in the video compositor. Empty/undefined
|
|
700
|
+
* for non-TYPE opcodes and for typing paths that bypass humanType.
|
|
701
|
+
*/
|
|
702
|
+
keystrokeOffsetsMs?: number[];
|
|
703
|
+
/**
|
|
704
|
+
* For CLICK / DOUBLE_CLICK / CHECK opcodes captured in clipCursor mode:
|
|
705
|
+
* timestamp (ms relative to the active clip start) at which Playwright
|
|
706
|
+
* dispatched each actual click — measured AFTER the cursor animation
|
|
707
|
+
* settled on the target. Drives mouse SFX in sync with the visible click.
|
|
708
|
+
* Empty/undefined for opcodes whose adapter doesn't surface the timestamp.
|
|
709
|
+
*/
|
|
710
|
+
clickOffsetsMs?: number[];
|
|
696
711
|
}
|
|
697
712
|
export interface RunResult {
|
|
698
713
|
programId: string;
|
|
@@ -729,6 +744,25 @@ export interface ClickOptions {
|
|
|
729
744
|
};
|
|
730
745
|
/** Mouse button. Default: 'left' */
|
|
731
746
|
button?: 'left' | 'right' | 'middle';
|
|
747
|
+
/**
|
|
748
|
+
* Fired with `Date.now()` right before Playwright dispatches the actual
|
|
749
|
+
* click — i.e. AFTER the visible cursor animation has settled on the
|
|
750
|
+
* target. The runner converts the wall-clock to clip-relative offsets so
|
|
751
|
+
* the video compositor can fire mouse SFX in lock-step with the visible
|
|
752
|
+
* click (instead of when the cursor was still travelling).
|
|
753
|
+
*/
|
|
754
|
+
onClick?: (timestampMs: number) => void;
|
|
755
|
+
}
|
|
756
|
+
export interface ClickByTargetOptions {
|
|
757
|
+
selector?: string;
|
|
758
|
+
target?: SemanticTarget;
|
|
759
|
+
selectorAlternates?: string[];
|
|
760
|
+
onClick?: (timestampMs: number) => void;
|
|
761
|
+
}
|
|
762
|
+
export interface MouseActionOptions {
|
|
763
|
+
/** Same semantics as `ClickOptions.onClick` — fires right before the
|
|
764
|
+
* actual click is dispatched (CHECK / DOUBLE_CLICK). */
|
|
765
|
+
onClick?: (timestampMs: number) => void;
|
|
732
766
|
}
|
|
733
767
|
export interface RecordingOptions {
|
|
734
768
|
mediaMode: 'clip' | 'video';
|
|
@@ -749,13 +783,23 @@ export interface RecordingResult {
|
|
|
749
783
|
mimeType: string;
|
|
750
784
|
trimStartMs?: number;
|
|
751
785
|
}
|
|
786
|
+
export interface TypeOptions {
|
|
787
|
+
/**
|
|
788
|
+
* Called once per keystroke produced by `humanType`, with the absolute
|
|
789
|
+
* wall-clock timestamp (`Date.now()`) of the keystroke. The runner converts
|
|
790
|
+
* those to clip-relative offsets stored on `OpcodeTiming.keystrokeOffsetsMs`
|
|
791
|
+
* so the compositor can fire per-keystroke SFX. Only fires in clipCursor
|
|
792
|
+
* mode (the only path that produces visible per-key animation).
|
|
793
|
+
*/
|
|
794
|
+
onKeystroke?: (timestampMs: number) => void;
|
|
795
|
+
}
|
|
752
796
|
export interface RuntimeAdapter {
|
|
753
797
|
navigate(url: string): Promise<void>;
|
|
754
798
|
getCurrentUrl(): Promise<string>;
|
|
755
799
|
getAKTree(): Promise<AKTree>;
|
|
756
800
|
getPageSignals(): Promise<VideoPageSignals>;
|
|
757
801
|
click(selector: string, options?: ClickOptions): Promise<void>;
|
|
758
|
-
type(selector: string, text: string, clearFirst?: boolean): Promise<void>;
|
|
802
|
+
type(selector: string, text: string, clearFirst?: boolean, opts?: TypeOptions): Promise<void>;
|
|
759
803
|
pressKey(key: string): Promise<void>;
|
|
760
804
|
scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise<void>;
|
|
761
805
|
scrollIntoView(selector: string): Promise<void>;
|
|
@@ -798,17 +842,13 @@ export interface RuntimeAdapter {
|
|
|
798
842
|
} | null>;
|
|
799
843
|
close(): Promise<void>;
|
|
800
844
|
/** Click an element by semantic target. Falls back to selector if target not found. */
|
|
801
|
-
clickByTarget?(opts:
|
|
802
|
-
selector?: string;
|
|
803
|
-
target?: SemanticTarget;
|
|
804
|
-
selectorAlternates?: string[];
|
|
805
|
-
}): Promise<void>;
|
|
845
|
+
clickByTarget?(opts: ClickByTargetOptions): Promise<void>;
|
|
806
846
|
/** Type into an element by semantic target. */
|
|
807
847
|
typeByTarget?(opts: {
|
|
808
848
|
selector?: string;
|
|
809
849
|
target?: SemanticTarget;
|
|
810
850
|
selectorAlternates?: string[];
|
|
811
|
-
}, text: string, clearFirst?: boolean): Promise<void>;
|
|
851
|
+
}, text: string, clearFirst?: boolean, typeOpts?: TypeOptions): Promise<void>;
|
|
812
852
|
/** Wait for an element by semantic target. */
|
|
813
853
|
waitForTarget?(opts: {
|
|
814
854
|
selector?: string;
|
|
@@ -832,8 +872,8 @@ export interface RuntimeAdapter {
|
|
|
832
872
|
value?: string;
|
|
833
873
|
index?: number;
|
|
834
874
|
}): Promise<void>;
|
|
835
|
-
check?(selector: string, checked: boolean): Promise<void>;
|
|
836
|
-
doubleClick?(selector: string): Promise<void>;
|
|
875
|
+
check?(selector: string, checked: boolean, opts?: MouseActionOptions): Promise<void>;
|
|
876
|
+
doubleClick?(selector: string, opts?: MouseActionOptions): Promise<void>;
|
|
837
877
|
/**
|
|
838
878
|
* Drag the source element from point A to point B with an animated cursor
|
|
839
879
|
* when a clip is recording. Destination is either another element
|
|
@@ -52,8 +52,14 @@ export declare function animatedHover(page: Page, target: {
|
|
|
52
52
|
/**
|
|
53
53
|
* Type text into the currently focused element at a human-like typing speed.
|
|
54
54
|
* Assumes the field is already focused (via a preceding click).
|
|
55
|
+
*
|
|
56
|
+
* `onKeystroke` fires after each character with the absolute wall-clock
|
|
57
|
+
* timestamp (`Date.now()`) of the keystroke. The video pipeline converts
|
|
58
|
+
* these to clip-relative offsets so keyboard SFX fire in lock-step with the
|
|
59
|
+
* visible typing.
|
|
55
60
|
*/
|
|
56
61
|
export declare function humanType(page: Page, text: string, options?: {
|
|
57
62
|
minDelayMs?: number;
|
|
58
63
|
maxDelayMs?: number;
|
|
64
|
+
onKeystroke?: (timestampMs: number) => void;
|
|
59
65
|
}): Promise<void>;
|
package/dist/mouse-animation.js
CHANGED
|
@@ -132,12 +132,20 @@ export async function animatedHover(page, target, fromCurrent, options = {}) {
|
|
|
132
132
|
/**
|
|
133
133
|
* Type text into the currently focused element at a human-like typing speed.
|
|
134
134
|
* Assumes the field is already focused (via a preceding click).
|
|
135
|
+
*
|
|
136
|
+
* `onKeystroke` fires after each character with the absolute wall-clock
|
|
137
|
+
* timestamp (`Date.now()`) of the keystroke. The video pipeline converts
|
|
138
|
+
* these to clip-relative offsets so keyboard SFX fire in lock-step with the
|
|
139
|
+
* visible typing.
|
|
135
140
|
*/
|
|
136
141
|
export async function humanType(page, text, options = {}) {
|
|
137
142
|
const minDelay = Math.max(0, options.minDelayMs ?? 60);
|
|
138
143
|
const maxDelay = Math.max(minDelay, options.maxDelayMs ?? 140);
|
|
139
144
|
for (const char of text) {
|
|
140
145
|
await page.keyboard.type(char);
|
|
146
|
+
if (options.onKeystroke) {
|
|
147
|
+
options.onKeystroke(Date.now());
|
|
148
|
+
}
|
|
141
149
|
// 60–120 WPM → ~80–130ms between characters (5 chars per word)
|
|
142
150
|
const delay = minDelay + Math.random() * (maxDelay - minDelay);
|
|
143
151
|
if (delay > 0) {
|
package/dist/opcode-actions.d.ts
CHANGED
|
@@ -40,5 +40,19 @@ export declare function findUnresolvedCredentialPlaceholders(text: string, crede
|
|
|
40
40
|
export interface OpcodeActionResult {
|
|
41
41
|
success: boolean;
|
|
42
42
|
error?: string;
|
|
43
|
+
/**
|
|
44
|
+
* For TYPE opcodes: absolute wall-clock timestamps (`Date.now()`) of each
|
|
45
|
+
* keystroke produced by `humanType`. The runner converts these to
|
|
46
|
+
* clip-relative offsets so the video compositor can fire per-keystroke SFX.
|
|
47
|
+
*/
|
|
48
|
+
keystrokeTimestampsMs?: number[];
|
|
49
|
+
/**
|
|
50
|
+
* For CLICK / DOUBLE_CLICK / CHECK opcodes: absolute wall-clock timestamps
|
|
51
|
+
* captured INSIDE the adapter, just before Playwright dispatches the
|
|
52
|
+
* actual click — i.e. AFTER the cursor animation has settled on the
|
|
53
|
+
* target. Lets the compositor place the mouse SFX in sync with the visible
|
|
54
|
+
* click instead of when the cursor was still travelling.
|
|
55
|
+
*/
|
|
56
|
+
clickTimestampsMs?: number[];
|
|
43
57
|
}
|
|
44
58
|
export declare function executeOpcodeCoreAction(opcode: ExecutionOpcode, adapter: RuntimeAdapter, context?: OpcodeActionContext): Promise<OpcodeActionResult>;
|
package/dist/opcode-actions.js
CHANGED
|
@@ -62,9 +62,13 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
|
|
|
62
62
|
case 'DISMISS_OVERLAYS':
|
|
63
63
|
await dismissAllOverlays(adapter);
|
|
64
64
|
break;
|
|
65
|
-
case 'CLICK':
|
|
65
|
+
case 'CLICK': {
|
|
66
|
+
const clickTimestampsMs = [];
|
|
67
|
+
const onClick = (timestampMs) => {
|
|
68
|
+
clickTimestampsMs.push(timestampMs);
|
|
69
|
+
};
|
|
66
70
|
try {
|
|
67
|
-
await adapter.click(opcode.selector, opcode.button ? { button: opcode.button } :
|
|
71
|
+
await adapter.click(opcode.selector, { ...(opcode.button ? { button: opcode.button } : {}), onClick });
|
|
68
72
|
}
|
|
69
73
|
catch (error) {
|
|
70
74
|
if (!opcode.target || !adapter.clickByTarget)
|
|
@@ -73,16 +77,22 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
|
|
|
73
77
|
selector: opcode.selector,
|
|
74
78
|
target: opcode.target,
|
|
75
79
|
selectorAlternates: opcode.selectorAlternates,
|
|
80
|
+
onClick,
|
|
76
81
|
});
|
|
77
82
|
}
|
|
78
|
-
|
|
83
|
+
return { success: true, clickTimestampsMs };
|
|
84
|
+
}
|
|
79
85
|
case 'TYPE': {
|
|
80
86
|
const rawText = (opcode.textByLocale && context.currentVariant?.locale
|
|
81
87
|
? opcode.textByLocale[context.currentVariant.locale] ?? opcode.text
|
|
82
88
|
: opcode.text);
|
|
83
89
|
const text = substituteCredentialPlaceholders(rawText, context.credentials);
|
|
90
|
+
const keystrokeTimestampsMs = [];
|
|
91
|
+
const onKeystroke = (timestampMs) => {
|
|
92
|
+
keystrokeTimestampsMs.push(timestampMs);
|
|
93
|
+
};
|
|
84
94
|
try {
|
|
85
|
-
await adapter.type(opcode.selector, text, opcode.clearFirst);
|
|
95
|
+
await adapter.type(opcode.selector, text, opcode.clearFirst, { onKeystroke });
|
|
86
96
|
}
|
|
87
97
|
catch (error) {
|
|
88
98
|
if (!opcode.target || !adapter.typeByTarget)
|
|
@@ -91,9 +101,9 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
|
|
|
91
101
|
selector: opcode.selector,
|
|
92
102
|
target: opcode.target,
|
|
93
103
|
selectorAlternates: opcode.selectorAlternates,
|
|
94
|
-
}, text, opcode.clearFirst);
|
|
104
|
+
}, text, opcode.clearFirst, { onKeystroke });
|
|
95
105
|
}
|
|
96
|
-
|
|
106
|
+
return { success: true, keystrokeTimestampsMs };
|
|
97
107
|
}
|
|
98
108
|
case 'PRESS_KEY':
|
|
99
109
|
await adapter.pressKey(opcode.key);
|
|
@@ -170,16 +180,24 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
|
|
|
170
180
|
index: opcode.optionIndex,
|
|
171
181
|
});
|
|
172
182
|
break;
|
|
173
|
-
case 'CHECK':
|
|
183
|
+
case 'CHECK': {
|
|
174
184
|
if (!adapter.check)
|
|
175
185
|
return { success: false, error: 'adapter does not support CHECK' };
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
186
|
+
const clickTimestampsMs = [];
|
|
187
|
+
await adapter.check(opcode.selector, opcode.checked, {
|
|
188
|
+
onClick: (timestampMs) => clickTimestampsMs.push(timestampMs),
|
|
189
|
+
});
|
|
190
|
+
return { success: true, clickTimestampsMs };
|
|
191
|
+
}
|
|
192
|
+
case 'DOUBLE_CLICK': {
|
|
179
193
|
if (!adapter.doubleClick)
|
|
180
194
|
return { success: false, error: 'adapter does not support DOUBLE_CLICK' };
|
|
181
|
-
|
|
182
|
-
|
|
195
|
+
const clickTimestampsMs = [];
|
|
196
|
+
await adapter.doubleClick(opcode.selector, {
|
|
197
|
+
onClick: (timestampMs) => clickTimestampsMs.push(timestampMs),
|
|
198
|
+
});
|
|
199
|
+
return { success: true, clickTimestampsMs };
|
|
200
|
+
}
|
|
183
201
|
case 'DRAG':
|
|
184
202
|
if (!adapter.drag)
|
|
185
203
|
return { success: false, error: 'adapter does not support DRAG' };
|
package/dist/opcode-runner.js
CHANGED
|
@@ -287,6 +287,12 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
287
287
|
const result = await withTimeout(() => executeOpcodeAction(opcode, index, adapter, artifacts, telemetry, currentVariant, executionState, artifactPlan, mockDataGroups, options, credentials), actionBudgetMs);
|
|
288
288
|
logger.debug(`[opcode ${index}] action exec end — took ${Date.now() - actionStart}ms, success=${result.success}${result.error ? `, error=${result.error}` : ''}`);
|
|
289
289
|
if (preTiming) {
|
|
290
|
+
const keystrokeOffsetsMs = result.keystrokeTimestampsMs && result.keystrokeTimestampsMs.length > 0
|
|
291
|
+
? result.keystrokeTimestampsMs.map((t) => Math.max(0, t - preTiming.clipStartedAt))
|
|
292
|
+
: undefined;
|
|
293
|
+
const clickOffsetsMs = result.clickTimestampsMs && result.clickTimestampsMs.length > 0
|
|
294
|
+
? result.clickTimestampsMs.map((t) => Math.max(0, t - preTiming.clipStartedAt))
|
|
295
|
+
: undefined;
|
|
290
296
|
opcodeTimings.push({
|
|
291
297
|
stepIndex: index,
|
|
292
298
|
stepId: opcode.stepId,
|
|
@@ -296,6 +302,8 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
296
302
|
timecodeStartMs: preTiming.timecodeStartMs,
|
|
297
303
|
timecodeEndMs: Math.max(0, Date.now() - preTiming.clipStartedAt),
|
|
298
304
|
bbox: preTiming.bbox,
|
|
305
|
+
...(keystrokeOffsetsMs ? { keystrokeOffsetsMs } : {}),
|
|
306
|
+
...(clickOffsetsMs ? { clickOffsetsMs } : {}),
|
|
299
307
|
});
|
|
300
308
|
}
|
|
301
309
|
if (!result.success) {
|
|
@@ -662,6 +670,12 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
|
|
|
662
670
|
}
|
|
663
671
|
case 'END_CLIP': {
|
|
664
672
|
const clipIdentity = resolveClipIdentity(executionState.activeClip, opcode);
|
|
673
|
+
// Capture the URL BEFORE endRecording(): the local CDP branch of the
|
|
674
|
+
// adapter closes the browser context inside endRecording() to release
|
|
675
|
+
// the CDP session, which makes any subsequent browser operation throw
|
|
676
|
+
// "Browser not launched". The clip ends immediately before this call,
|
|
677
|
+
// so the URL is still accurate.
|
|
678
|
+
const captureUrl = await adapter.getCurrentUrl();
|
|
665
679
|
const recording = await adapter.endRecording();
|
|
666
680
|
executionState.activeClip = undefined;
|
|
667
681
|
// Match the artifact's mediaMode to the program's so the upload route
|
|
@@ -676,7 +690,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
|
|
|
676
690
|
trimStartMs: recording.trimStartMs,
|
|
677
691
|
dimensions: undefined,
|
|
678
692
|
captureType: 'fullpage',
|
|
679
|
-
captureUrl
|
|
693
|
+
captureUrl,
|
|
680
694
|
clipId: clipIdentity.clipId,
|
|
681
695
|
clipName: clipIdentity.clipName,
|
|
682
696
|
stepDescription: opcode.description,
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Preflight Playwright Chromium binary install.
|
|
3
|
+
*
|
|
4
|
+
* Run before any `chromium.launch(...)` to make sure the Playwright Chromium
|
|
5
|
+
* binary exists on disk. The package ships a `postinstall` hook that fetches
|
|
6
|
+
* Chromium at install time, but that step can be silently skipped on some
|
|
7
|
+
* setups (CI caches, monorepos, npm scripts) — and any Playwright version
|
|
8
|
+
* bump downstream invalidates the existing binary too. Catching this once at
|
|
9
|
+
* run start avoids the famously confusing "Executable doesn't exist at …"
|
|
10
|
+
* launch failure mid-capture.
|
|
11
|
+
*
|
|
12
|
+
* Cross-platform: uses `npx playwright install chromium` (works on macOS,
|
|
13
|
+
* Linux, Windows). Output is streamed inherit so the user sees the
|
|
14
|
+
* download progress.
|
|
15
|
+
*/
|
|
16
|
+
export declare function ensureChromiumInstalled(): Promise<void>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Preflight Playwright Chromium binary install.
|
|
3
|
+
*
|
|
4
|
+
* Run before any `chromium.launch(...)` to make sure the Playwright Chromium
|
|
5
|
+
* binary exists on disk. The package ships a `postinstall` hook that fetches
|
|
6
|
+
* Chromium at install time, but that step can be silently skipped on some
|
|
7
|
+
* setups (CI caches, monorepos, npm scripts) — and any Playwright version
|
|
8
|
+
* bump downstream invalidates the existing binary too. Catching this once at
|
|
9
|
+
* run start avoids the famously confusing "Executable doesn't exist at …"
|
|
10
|
+
* launch failure mid-capture.
|
|
11
|
+
*
|
|
12
|
+
* Cross-platform: uses `npx playwright install chromium` (works on macOS,
|
|
13
|
+
* Linux, Windows). Output is streamed inherit so the user sees the
|
|
14
|
+
* download progress.
|
|
15
|
+
*/
|
|
16
|
+
import { spawn } from 'node:child_process';
|
|
17
|
+
import { existsSync } from 'node:fs';
|
|
18
|
+
import { chromium } from 'playwright';
|
|
19
|
+
import { logger } from './logger.js';
|
|
20
|
+
let cachedCheckResult = null;
|
|
21
|
+
export async function ensureChromiumInstalled() {
|
|
22
|
+
// One-shot per process: once we've verified (or installed) the binary,
|
|
23
|
+
// skip every subsequent call. `chromium.executablePath()` is cheap but
|
|
24
|
+
// `existsSync` adds up across recovery retries.
|
|
25
|
+
if (cachedCheckResult === 'ok')
|
|
26
|
+
return;
|
|
27
|
+
let execPath = '';
|
|
28
|
+
try {
|
|
29
|
+
execPath = chromium.executablePath();
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// executablePath() throws if Playwright has no path resolved at all.
|
|
33
|
+
// Treat the same as "not installed" so we run the install.
|
|
34
|
+
execPath = '';
|
|
35
|
+
}
|
|
36
|
+
if (execPath && existsSync(execPath)) {
|
|
37
|
+
cachedCheckResult = 'ok';
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
logger.info('[playwright] Chromium browser is missing — installing it now (one-time, takes ~30s)…');
|
|
41
|
+
await runPlaywrightInstall();
|
|
42
|
+
// Re-check; if the install genuinely succeeded `executablePath()` now
|
|
43
|
+
// resolves to a real file. If it still doesn't, surface a clearer error
|
|
44
|
+
// than the raw launch failure.
|
|
45
|
+
try {
|
|
46
|
+
const refreshed = chromium.executablePath();
|
|
47
|
+
if (existsSync(refreshed)) {
|
|
48
|
+
cachedCheckResult = 'ok';
|
|
49
|
+
logger.info('[playwright] Chromium installed.');
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// fallthrough to error below
|
|
55
|
+
}
|
|
56
|
+
throw new Error('Playwright Chromium install completed but the binary is still missing. ' +
|
|
57
|
+
'Run `npx playwright install chromium` manually to diagnose.');
|
|
58
|
+
}
|
|
59
|
+
async function runPlaywrightInstall() {
|
|
60
|
+
return new Promise((resolve, reject) => {
|
|
61
|
+
// Use `npx` so we hit whichever Playwright version this CLI depends on,
|
|
62
|
+
// regardless of whether the user has a global `playwright` binary.
|
|
63
|
+
const child = spawn('npx', ['--yes', 'playwright', 'install', 'chromium'], {
|
|
64
|
+
stdio: 'inherit',
|
|
65
|
+
});
|
|
66
|
+
child.on('error', (err) => {
|
|
67
|
+
reject(new Error(`Failed to spawn \`npx playwright install chromium\`: ${err.message}. ` +
|
|
68
|
+
'Make sure Node.js + npm are installed and on PATH.'));
|
|
69
|
+
});
|
|
70
|
+
child.on('close', (code) => {
|
|
71
|
+
if (code === 0) {
|
|
72
|
+
resolve();
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
reject(new Error(`\`npx playwright install chromium\` exited with code ${code}. ` +
|
|
76
|
+
'Run it manually for full diagnostics.'));
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=playwright-installer.js.map
|
|
@@ -34,11 +34,15 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
|
|
|
34
34
|
* Click an element using semantic target resolution.
|
|
35
35
|
* Tries CSS selector first, falls back to Playwright semantic locators.
|
|
36
36
|
*/
|
|
37
|
-
clickByTarget(opts: ResolveOptions
|
|
37
|
+
clickByTarget(opts: ResolveOptions & {
|
|
38
|
+
onClick?: (timestampMs: number) => void;
|
|
39
|
+
}): Promise<void>;
|
|
38
40
|
/**
|
|
39
41
|
* Type into an element using semantic target resolution.
|
|
40
42
|
*/
|
|
41
|
-
typeByTarget(opts: ResolveOptions, text: string, clearFirst?: boolean
|
|
43
|
+
typeByTarget(opts: ResolveOptions, text: string, clearFirst?: boolean, typeOpts?: {
|
|
44
|
+
onKeystroke?: (timestampMs: number) => void;
|
|
45
|
+
}): Promise<void>;
|
|
42
46
|
/**
|
|
43
47
|
* Wait for an element using semantic target resolution.
|
|
44
48
|
*/
|
|
@@ -47,7 +51,9 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
|
|
|
47
51
|
* Scroll an element into view using semantic target resolution.
|
|
48
52
|
*/
|
|
49
53
|
scrollIntoViewByTarget(opts: ResolveOptions): Promise<void>;
|
|
50
|
-
type(selector: string, text: string, clearFirst?: boolean
|
|
54
|
+
type(selector: string, text: string, clearFirst?: boolean, opts?: {
|
|
55
|
+
onKeystroke?: (timestampMs: number) => void;
|
|
56
|
+
}): Promise<void>;
|
|
51
57
|
pressKey(key: string): Promise<void>;
|
|
52
58
|
scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise<void>;
|
|
53
59
|
scrollIntoView(selector: string): Promise<void>;
|
|
@@ -83,8 +89,12 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
|
|
|
83
89
|
value?: string;
|
|
84
90
|
index?: number;
|
|
85
91
|
}): Promise<void>;
|
|
86
|
-
check(selector: string, checked: boolean
|
|
87
|
-
|
|
92
|
+
check(selector: string, checked: boolean, actionOpts?: {
|
|
93
|
+
onClick?: (timestampMs: number) => void;
|
|
94
|
+
}): Promise<void>;
|
|
95
|
+
doubleClick(selector: string, actionOpts?: {
|
|
96
|
+
onClick?: (timestampMs: number) => void;
|
|
97
|
+
}): Promise<void>;
|
|
88
98
|
drag(opts: {
|
|
89
99
|
selector?: string;
|
|
90
100
|
target?: SemanticTarget;
|
|
@@ -77,9 +77,11 @@ export class WebPlaywrightLocal {
|
|
|
77
77
|
const page = await this.browser.currentPage;
|
|
78
78
|
const t0 = Date.now();
|
|
79
79
|
logger.debug(`[click] start selector="${selector}"${options?.useKeyboard ? ' mode=keyboard' : ''}${options?.useJsDispatch ? ' mode=js_dispatch' : ''}${options?.coordinates ? ` mode=coords(${options.coordinates.x},${options.coordinates.y})` : ''}`);
|
|
80
|
+
const fireClickSfx = () => options?.onClick?.(Date.now());
|
|
80
81
|
try {
|
|
81
82
|
if (options?.coordinates) {
|
|
82
83
|
await this.moveClipCursorToPoint(options.coordinates);
|
|
84
|
+
fireClickSfx();
|
|
83
85
|
await this.browser.clickByCoordinates(options.coordinates.x, options.coordinates.y);
|
|
84
86
|
logger.debug(`[click] done coords took ${Date.now() - t0}ms`);
|
|
85
87
|
return;
|
|
@@ -88,11 +90,13 @@ export class WebPlaywrightLocal {
|
|
|
88
90
|
const animatedTarget = await this.moveClipCursorToLocator(locator);
|
|
89
91
|
if (options?.useKeyboard) {
|
|
90
92
|
await locator.focus();
|
|
93
|
+
fireClickSfx();
|
|
91
94
|
await page.keyboard.press('Enter');
|
|
92
95
|
logger.debug(`[click] done keyboard took ${Date.now() - t0}ms`);
|
|
93
96
|
return;
|
|
94
97
|
}
|
|
95
98
|
if (options?.useJsDispatch) {
|
|
99
|
+
fireClickSfx();
|
|
96
100
|
await locator.dispatchEvent('click');
|
|
97
101
|
logger.debug(`[click] done js_dispatch took ${Date.now() - t0}ms`);
|
|
98
102
|
return;
|
|
@@ -103,6 +107,7 @@ export class WebPlaywrightLocal {
|
|
|
103
107
|
? await this.relativeClickPosition(locator, animatedTarget)
|
|
104
108
|
: null;
|
|
105
109
|
if (options?.button && options.button !== 'left') {
|
|
110
|
+
fireClickSfx();
|
|
106
111
|
await locator.click({
|
|
107
112
|
button: options.button,
|
|
108
113
|
timeout: 5000,
|
|
@@ -113,6 +118,7 @@ export class WebPlaywrightLocal {
|
|
|
113
118
|
return;
|
|
114
119
|
}
|
|
115
120
|
if (clickPosition) {
|
|
121
|
+
fireClickSfx();
|
|
116
122
|
await locator.click({
|
|
117
123
|
timeout: 5000,
|
|
118
124
|
force: options?.force,
|
|
@@ -120,6 +126,7 @@ export class WebPlaywrightLocal {
|
|
|
120
126
|
});
|
|
121
127
|
}
|
|
122
128
|
else {
|
|
129
|
+
fireClickSfx();
|
|
123
130
|
await this.browser.clickBySelector(selector, { force: options?.force });
|
|
124
131
|
}
|
|
125
132
|
await this.emitClipClickPulse();
|
|
@@ -144,6 +151,7 @@ export class WebPlaywrightLocal {
|
|
|
144
151
|
const position = target
|
|
145
152
|
? await this.relativeClickPosition(resolved.locator, target)
|
|
146
153
|
: null;
|
|
154
|
+
opts.onClick?.(Date.now());
|
|
147
155
|
await resolved.locator.click({
|
|
148
156
|
timeout: 5000,
|
|
149
157
|
...(position ? { position } : {}),
|
|
@@ -152,14 +160,14 @@ export class WebPlaywrightLocal {
|
|
|
152
160
|
/**
|
|
153
161
|
* Type into an element using semantic target resolution.
|
|
154
162
|
*/
|
|
155
|
-
async typeByTarget(opts, text, clearFirst = true) {
|
|
163
|
+
async typeByTarget(opts, text, clearFirst = true, typeOpts) {
|
|
156
164
|
const page = await this.browser.currentPage;
|
|
157
165
|
const resolved = await resolveTarget(page, opts);
|
|
158
166
|
if (!resolved) {
|
|
159
167
|
throw new Error(`cannot find target for typing: ${describeResolveOptions(opts)}`);
|
|
160
168
|
}
|
|
161
169
|
if (this.clipCursor) {
|
|
162
|
-
await this.typeIntoLocator(resolved.locator, text, clearFirst);
|
|
170
|
+
await this.typeIntoLocator(resolved.locator, text, clearFirst, typeOpts?.onKeystroke);
|
|
163
171
|
return;
|
|
164
172
|
}
|
|
165
173
|
if (clearFirst) {
|
|
@@ -196,10 +204,10 @@ export class WebPlaywrightLocal {
|
|
|
196
204
|
}
|
|
197
205
|
await resolved.locator.scrollIntoViewIfNeeded({ timeout: 5000 });
|
|
198
206
|
}
|
|
199
|
-
async type(selector, text, clearFirst = true) {
|
|
207
|
+
async type(selector, text, clearFirst = true, opts) {
|
|
200
208
|
if (this.clipCursor) {
|
|
201
209
|
const page = await this.browser.currentPage;
|
|
202
|
-
await this.typeIntoLocator(page.locator(selector).first(), text, clearFirst);
|
|
210
|
+
await this.typeIntoLocator(page.locator(selector).first(), text, clearFirst, opts?.onKeystroke);
|
|
203
211
|
return;
|
|
204
212
|
}
|
|
205
213
|
await this.browser.typeText(text, { selector, clearFirst });
|
|
@@ -633,7 +641,7 @@ export class WebPlaywrightLocal {
|
|
|
633
641
|
optionIndex: option.index,
|
|
634
642
|
});
|
|
635
643
|
}
|
|
636
|
-
async check(selector, checked) {
|
|
644
|
+
async check(selector, checked, actionOpts) {
|
|
637
645
|
const page = await this.browser.currentPage;
|
|
638
646
|
const locator = page.locator(selector).first();
|
|
639
647
|
const target = await this.moveClipCursorToLocator(locator);
|
|
@@ -641,6 +649,7 @@ export class WebPlaywrightLocal {
|
|
|
641
649
|
? await this.relativeClickPosition(locator, target)
|
|
642
650
|
: null;
|
|
643
651
|
const opts = { timeout: 5000, ...(position ? { position } : {}) };
|
|
652
|
+
actionOpts?.onClick?.(Date.now());
|
|
644
653
|
if (checked) {
|
|
645
654
|
await locator.check(opts);
|
|
646
655
|
}
|
|
@@ -648,13 +657,14 @@ export class WebPlaywrightLocal {
|
|
|
648
657
|
await locator.uncheck(opts);
|
|
649
658
|
}
|
|
650
659
|
}
|
|
651
|
-
async doubleClick(selector) {
|
|
660
|
+
async doubleClick(selector, actionOpts) {
|
|
652
661
|
const page = await this.browser.currentPage;
|
|
653
662
|
const locator = page.locator(selector).first();
|
|
654
663
|
const target = await this.moveClipCursorToLocator(locator);
|
|
655
664
|
const position = target
|
|
656
665
|
? await this.relativeClickPosition(locator, target)
|
|
657
666
|
: null;
|
|
667
|
+
actionOpts?.onClick?.(Date.now());
|
|
658
668
|
await locator.dblclick({
|
|
659
669
|
timeout: 5000,
|
|
660
670
|
...(position ? { position } : {}),
|
|
@@ -878,7 +888,7 @@ export class WebPlaywrightLocal {
|
|
|
878
888
|
async close() {
|
|
879
889
|
await this.browser.close();
|
|
880
890
|
}
|
|
881
|
-
async typeIntoLocator(locator, text, clearFirst) {
|
|
891
|
+
async typeIntoLocator(locator, text, clearFirst, onKeystroke) {
|
|
882
892
|
const page = await this.browser.currentPage;
|
|
883
893
|
await locator.waitFor({ state: 'visible', timeout: 5000 });
|
|
884
894
|
await locator.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => undefined);
|
|
@@ -895,8 +905,11 @@ export class WebPlaywrightLocal {
|
|
|
895
905
|
}
|
|
896
906
|
await page.waitForTimeout(70);
|
|
897
907
|
await humanType(page, text, this.clipCursor
|
|
898
|
-
|
|
899
|
-
|
|
908
|
+
// Demo-video cadence: ~80-180ms between keys (≈ 80-100 WPM with
|
|
909
|
+
// natural variation). Faster than that reads as robotic in the
|
|
910
|
+
// mixed video + keyboard SFX track.
|
|
911
|
+
? { minDelayMs: 80, maxDelayMs: 180, onKeystroke }
|
|
912
|
+
: { onKeystroke });
|
|
900
913
|
}
|
|
901
914
|
async seedClipCursor(position) {
|
|
902
915
|
if (!this.clipCursor)
|