@midscene/cli 1.5.6 → 1.5.7-beta-20260317091411.0

package/dist/es/index.mjs

@@ -19,6 +19,11 @@ import { EventEmitter } from "node:events";
 import node_stream from "node:stream";
 import { StringDecoder } from "node:string_decoder";
 import { fileURLToPath as external_url_fileURLToPath } from "url";
+import { generatePlaywrightFromVideoFrames, generateYamlFromVideoFrames } from "@midscene/core/ai-model";
+import { globalModelConfigManager } from "@midscene/shared/env";
+import { spawnSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
 import * as __rspack_external_assert from "assert";
 import * as __rspack_external_crypto from "crypto";
 import * as __rspack_external_fs from "fs";
@@ -3022,9 +3027,7 @@ var __webpack_modules__ = {
     "./src/index.ts" (__unused_rspack_module, __unused_rspack___webpack_exports__, __webpack_require__) {
         var main = __webpack_require__("../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/lib/main.js");
         var main_default = /*#__PURE__*/ __webpack_require__.n(main);
-        var package_namespaceObject = {
-            rE: "1.5.6"
-        };
+        var package_namespaceObject = JSON.parse('{"rE":"1.5.7-beta-20260317091411.0"}');
         class Node {
             value;
             next;
@@ -11924,7 +11927,7 @@ Usage:
                     type: 'boolean',
                     description: `Turn on logging to help debug why certain keys or values are not being set as you expect, default is ${config_factory_defaultConfig.dotenvDebug}`
                 }
-            }).version('version', 'Show version number', "1.5.6").help().epilogue(`For complete list of configuration options, please visit:
+            }).version('version', 'Show version number', "1.5.7-beta-20260317091411.0").help().epilogue(`For complete list of configuration options, please visit:
   • Web options: https://midscenejs.com/automate-with-scripts-in-yaml#the-web-part
   • Android options: https://midscenejs.com/automate-with-scripts-in-yaml#the-android-part
   • iOS options: https://midscenejs.com/automate-with-scripts-in-yaml#the-ios-part
@@ -11972,7 +11975,272 @@ Examples:
             });
             return files.filter((file)=>file.endsWith('.yml') || file.endsWith('.yaml')).sort();
         }
+        function consumeArg(args, index, flag) {
+            const value = args[index + 1];
+            if (void 0 === value || value.startsWith('-')) {
+                console.error(`Error: ${flag} requires a value`);
+                process.exit(1);
+            }
+            return value;
+        }
+        function parseVideo2YamlArgs(args) {
+            const options = {
+                input: ''
+            };
+            let i = 0;
+            while(i < args.length){
+                const arg = args[i];
+                if ('--output' === arg || '-o' === arg) {
+                    options.output = consumeArg(args, i, arg);
+                    i++;
+                } else if ('--format' === arg || '-f' === arg) {
+                    const fmt = consumeArg(args, i, arg);
+                    if ('yaml' !== fmt && 'playwright' !== fmt) {
+                        console.error(`Invalid format: ${fmt}. Must be "yaml" or "playwright"`);
+                        process.exit(1);
+                    }
+                    options.format = fmt;
+                    i++;
+                } else if ('--url' === arg) {
+                    options.url = consumeArg(args, i, arg);
+                    i++;
+                } else if ("--description" === arg) {
+                    options.description = consumeArg(args, i, arg);
+                    i++;
+                } else if ('--fps' === arg) {
+                    options.fps = Number(consumeArg(args, i, arg));
+                    i++;
+                } else if ('--max-frames' === arg) {
+                    options.maxFrames = Number(consumeArg(args, i, arg));
+                    i++;
+                } else if ('--viewport-width' === arg) {
+                    options.viewportWidth = Number(consumeArg(args, i, arg));
+                    i++;
+                } else if ('--viewport-height' === arg) {
+                    options.viewportHeight = Number(consumeArg(args, i, arg));
+                    i++;
+                } else if ('--help' === arg || '-h' === arg) {
+                    printHelp();
+                    process.exit(0);
+                } else if (arg.startsWith('-') || options.input) {
+                    console.error(`Unknown option: ${arg}`);
+                    printHelp();
+                    process.exit(1);
+                } else options.input = arg;
+                i++;
+            }
+            if (!options.input) {
+                console.error('Error: video file path is required');
+                printHelp();
+                process.exit(1);
+            }
+            return options;
+        }
+        function printHelp() {
+            console.log(`
+Usage: midscene video2yaml <video-file> [options]
+
+Generate a runnable Midscene test script from a screen recording video.
+
+Arguments:
+  video-file              Path to the video file (mp4, webm, etc.)
+
+Options:
+  -o, --output <path>     Output file path (default: <video-file>.yaml or .test.ts)
+  -f, --format <format>   Output format: "yaml" (default) or "playwright"
+  --url <url>             Starting URL of the web page in the video
+  --description <text>    Description of what the video demonstrates
+  --fps <number>          Frames per second to extract (default: 1)
+  --max-frames <number>   Maximum number of frames to analyze (default: 20)
+  --viewport-width <px>   Viewport width of the recorded page
+  --viewport-height <px>  Viewport height of the recorded page
+  -h, --help              Show this help message
+
+Examples:
+  # Generate YAML script (default)
+  midscene video2yaml recording.mp4
+  midscene video2yaml recording.mp4 -o test.yaml --url https://example.com
+
+  # Generate Playwright test
+  midscene video2yaml recording.mp4 --format playwright
+  midscene video2yaml recording.mp4 -f playwright -o login.test.ts
+
+  # Custom frame extraction
+  midscene video2yaml demo.webm --fps 2 --max-frames 30 --description "Login flow test"
+`);
+        }
+        const extract_frames_debug = getDebug('cli:video');
+        function resolveExecutable(npmPackage, systemFallback) {
+            try {
+                const dynamicRequire = createRequire(__filename);
+                const installer = dynamicRequire(npmPackage);
+                const binPath = installer.path;
+                try {
+                    (0, __rspack_external_node_fs_5ea92f0c.chmodSync)(binPath, 493);
+                } catch  {}
+                const check = spawnSync(binPath, [
+                    '-version'
+                ], {
+                    stdio: 'pipe',
+                    timeout: 5000
+                });
+                if (check.error || 0 !== check.status) {
+                    extract_frames_debug(`npm ${npmPackage} binary not executable (${check.error?.message ?? `status ${check.status}`}), falling back to system ${systemFallback}`);
+                    return systemFallback;
+                }
+                extract_frames_debug(`Using ${systemFallback} from npm package: ${binPath}`);
+                return binPath;
+            } catch (error) {
+                extract_frames_debug(`npm ${npmPackage} not found (${error}), falling back to system ${systemFallback}`);
+                return systemFallback;
+            }
+        }
+        function getFfmpegPath() {
+            return resolveExecutable('@ffmpeg-installer/ffmpeg', 'ffmpeg');
+        }
+        function getFfprobePath() {
+            return resolveExecutable('@ffprobe-installer/ffprobe', 'ffprobe');
+        }
+        function checkFfmpeg() {
+            try {
+                const ffmpegPath = getFfmpegPath();
+                const result = spawnSync(ffmpegPath, [
+                    '-version'
+                ], {
+                    stdio: 'pipe',
+                    timeout: 5000
+                });
+                extract_frames_debug(`ffmpeg check result: status=${result.status}`);
+                return 0 === result.status;
+            } catch (error) {
+                extract_frames_debug(`ffmpeg check failed: ${error}`);
+                return false;
+            }
+        }
+        function getVideoDuration(videoPath) {
+            const ffprobePath = getFfprobePath();
+            const result = spawnSync(ffprobePath, [
+                '-v',
+                'error',
+                '-show_entries',
+                'format=duration',
+                '-of',
+                'csv=p=0',
+                videoPath
+            ], {
+                stdio: 'pipe',
+                timeout: 10000
+            });
+            if (result.error) throw new Error(`Failed to run ffprobe: ${result.error.message}`);
+            if (0 !== result.status) throw new Error(`Failed to get video duration: ${result.stderr?.toString() || 'unknown error'}`);
+            const duration = Number.parseFloat(result.stdout.toString().trim());
+            if (Number.isNaN(duration)) throw new Error('Could not parse video duration');
+            return duration;
+        }
+        function extractFrames(videoPath, options) {
+            const { fps = 1, maxFrames, width } = options;
+            if (!(0, __rspack_external_node_fs_5ea92f0c.existsSync)(videoPath)) throw new Error(`Video file not found: ${videoPath}`);
+            if (!checkFfmpeg()) throw new Error("ffmpeg is not available.\nTo fix this, either:\n  1. Install the npm package: pnpm add -D @ffmpeg-installer/ffmpeg\n  2. Or install system ffmpeg: https://ffmpeg.org/download.html");
+            const duration = getVideoDuration(videoPath);
+            const totalFramesAtFps = Math.ceil(duration * fps);
+            const actualFps = totalFramesAtFps > maxFrames ? maxFrames / duration : fps;
+            const tempDir = join(tmpdir(), `midscene-video-frames-${Date.now()}`);
+            (0, __rspack_external_node_fs_5ea92f0c.mkdirSync)(tempDir, {
+                recursive: true
+            });
+            try {
+                const filters = [
+                    `fps=${actualFps}`
+                ];
+                if (width) filters.push(`scale=${Math.min(width, 1920)}:-1`);
+                else filters.push('scale=min(iw\\,1920):-1');
+                const ffmpegPath = getFfmpegPath();
+                const ffmpegArgs = [
+                    '-i',
+                    videoPath,
+                    '-vf',
+                    filters.join(','),
+                    '-q:v',
+                    '2',
+                    '-frames:v',
+                    String(maxFrames),
+                    join(tempDir, 'frame_%04d.jpg')
+                ];
+                const result = spawnSync(ffmpegPath, ffmpegArgs, {
+                    stdio: 'pipe',
+                    timeout: 120000
+                });
+                if (result.error) throw new Error(`ffmpeg execution error: ${result.error.message}`);
+                if (0 !== result.status) throw new Error(`ffmpeg failed: ${result.stderr?.toString() || 'unknown error'}`);
+                const frameFiles = (0, __rspack_external_node_fs_5ea92f0c.readdirSync)(tempDir).filter((f)=>f.startsWith('frame_') && f.endsWith('.jpg')).sort();
+                const frames = [];
+                for(let i = 0; i < frameFiles.length; i++){
+                    const filePath = join(tempDir, frameFiles[i]);
+                    const buffer = (0, __rspack_external_node_fs_5ea92f0c.readFileSync)(filePath);
+                    const base64 = buffer.toString('base64');
+                    const timestamp = i / actualFps;
+                    frames.push({
+                        base64: `data:image/jpeg;base64,${base64}`,
+                        timestamp
+                    });
+                }
+                return frames;
+            } finally{
+                try {
+                    (0, __rspack_external_node_fs_5ea92f0c.rmSync)(tempDir, {
+                        recursive: true,
+                        force: true
+                    });
+                } catch  {}
+            }
+        }
+        function getDefaultOutputPath(inputPath, format) {
+            const ext = 'playwright' === format ? '.test.ts' : '.yaml';
+            return inputPath.replace(/\.[^.]+$/, ext);
+        }
+        async function video2yaml(options) {
+            const { input, output, format = 'yaml', url, description, fps = 1, maxFrames = 20, viewportWidth, viewportHeight } = options;
+            const inputPath = external_node_path_resolve(input);
+            if (!(0, __rspack_external_node_fs_5ea92f0c.existsSync)(inputPath)) throw new Error(`Video file not found: ${inputPath}`);
+            const outputPath = output ? external_node_path_resolve(output) : getDefaultOutputPath(inputPath, format);
+            console.log(`\n   Extracting frames from video (fps=${fps}, max=${maxFrames})...`);
+            const frames = extractFrames(inputPath, {
+                fps,
+                maxFrames
+            });
+            console.log(`   Extracted ${frames.length} frames`);
+            if (0 === frames.length) throw new Error('No frames could be extracted from the video');
+            const formatLabel = 'playwright' === format ? 'Playwright test' : 'YAML';
+            console.log(`   Analyzing video frames with AI (generating ${formatLabel})...`);
+            const modelConfig = globalModelConfigManager.getModelConfig('default');
+            const scriptOptions = {
+                url,
+                description,
+                viewportWidth,
+                viewportHeight
+            };
+            const result = 'playwright' === format ? await generatePlaywrightFromVideoFrames(frames, scriptOptions, modelConfig) : await generateYamlFromVideoFrames(frames, scriptOptions, modelConfig);
+            (0, __rspack_external_node_fs_5ea92f0c.writeFileSync)(outputPath, result.content, 'utf-8');
+            console.log(`   ${formatLabel} script saved to: ${outputPath}`);
+            return outputPath;
+        }
+        async function handleVideo2Yaml(args) {
+            const options = parseVideo2YamlArgs(args);
+            const dotEnvConfigFile = join(process.cwd(), '.env');
+            if ((0, __rspack_external_node_fs_5ea92f0c.existsSync)(dotEnvConfigFile)) main_default().config({
+                path: dotEnvConfigFile
+            });
+            await video2yaml(options);
+        }
         Promise.resolve((async ()=>{
+            const rawArgs = process.argv.slice(2);
+            if ('video2yaml' === rawArgs[0]) {
+                const welcome = `\nWelcome to @midscene/cli v${package_namespaceObject.rE}\n`;
+                console.log(welcome);
+                await handleVideo2Yaml(rawArgs.slice(1));
+                process.exit(0);
+                return;
+            }
             const { options, path, files: cmdFiles } = await parseProcessArgs();
             const welcome = `\nWelcome to @midscene/cli v${package_namespaceObject.rE}\n`;
             console.log(welcome);