@midscene/test 1.14.0 → 1.15.0

package/dist/lib/cli/model-command.js

@@ -0,0 +1,367 @@
+"use strict";
+var __webpack_modules__ = {
+    "@midscene/core" (module) {
+        module.exports = require("@midscene/core");
+    },
+    "@midscene/shared/env" (module) {
+        module.exports = require("@midscene/shared/env");
+    },
+    chalk (module) {
+        module.exports = require("chalk");
+    },
+    dotenv (module) {
+        module.exports = require("dotenv");
+    },
+    "node:fs" (module) {
+        module.exports = require("node:fs");
+    },
+    "node:path" (module) {
+        module.exports = require("node:path");
+    }
+};
+var __webpack_module_cache__ = {};
+function __webpack_require__(moduleId) {
+    var cachedModule = __webpack_module_cache__[moduleId];
+    if (void 0 !== cachedModule) return cachedModule.exports;
+    var module = __webpack_module_cache__[moduleId] = {
+        exports: {}
+    };
+    __webpack_modules__[moduleId](module, module.exports, __webpack_require__);
+    return module.exports;
+}
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+(()=>{
+    __webpack_require__.r(__webpack_exports__);
+    __webpack_require__.d(__webpack_exports__, {
+        buildModelVerifyCurlCommands: ()=>buildModelVerifyCurlCommands,
+        loadDotenvConfig: ()=>loadDotenvConfig,
+        runModelCommand: ()=>runModelCommand
+    });
+    var core_ = __webpack_require__("@midscene/core");
+    var env_ = __webpack_require__("@midscene/shared/env");
+    var external_chalk_ = __webpack_require__("chalk");
+    var external_chalk_default = /*#__PURE__*/ __webpack_require__.n(external_chalk_);
+    var external_node_fs_ = __webpack_require__("node:fs");
+    var external_node_path_ = __webpack_require__("node:path");
+    var external_dotenv_ = __webpack_require__("dotenv");
+    var external_dotenv_default = /*#__PURE__*/ __webpack_require__.n(external_dotenv_);
+    function loadDotenvConfig(options = {}) {
+        const dotEnvConfigFile = (0, external_node_path_.join)(options.cwd ?? process.cwd(), '.env');
+        if (!(0, external_node_fs_.existsSync)(dotEnvConfigFile)) return;
+        options.log?.(`   Env file: ${dotEnvConfigFile}`);
+        external_dotenv_default().config({
+            path: dotEnvConfigFile,
+            debug: options.dotenvDebug,
+            override: options.dotenvOverride
+        });
+    }
+    const DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1';
+    const MODEL_VERIFY_SEPARATOR = '────────────────────────────────────────';
+    const modelCommandUsage = (commandName)=>`Usage:
+  ${commandName} model verify
+
+Verify model connectivity and Midscene compatibility for default, planning, and insight models.
+Reads .env from the current working directory, overriding shell environment variables.
+Shows endpoints, model names, request counts, and per-check results with duration.
+Returns exit code 0 on success and 1 on failure.
+
+Options:
+  --help, -h  Show this help
+`;
+    function assertNoModelVerifyOptions(args, commandName) {
+        for (const arg of args)throw new Error(`Unknown option for ${commandName} model verify: ${arg}`);
+    }
+    function formatTable(headers, rows) {
+        const widths = headers.map((header, index)=>Math.max(header.length, ...rows.map((row)=>row[index].length)));
+        const formatRow = (row)=>row.map((cell, index)=>cell.padEnd(widths[index])).join('  ').trimEnd();
+        return [
+            formatRow(headers),
+            ...rows.map(formatRow)
+        ].join('\n');
+    }
+    function displayEndpoint(endpoint) {
+        try {
+            const url = new URL(endpoint);
+            url.username = '';
+            url.password = '';
+            url.search = '';
+            url.hash = '';
+            return url.toString();
+        } catch  {
+            return '(invalid endpoint URL)';
+        }
+    }
+    function shellSingleQuote(value) {
+        return `'${value.replace(/'/g, "'\\''")}'`;
+    }
+    function buildChatCompletionsUrl(baseURL) {
+        const normalizedBaseURL = (baseURL || DEFAULT_OPENAI_BASE_URL).replace(/\/+$/, '');
+        if (normalizedBaseURL.endsWith('/chat/completions')) return normalizedBaseURL;
+        return `${normalizedBaseURL}/chat/completions`;
+    }
+    function buildCurlCommand(modelConfig) {
+        const payload = {
+            model: modelConfig.modelName,
+            messages: [
+                {
+                    role: 'user',
+                    content: 'What is 1+1?'
+                }
+            ]
+        };
+        return [
+            `curl -X POST ${shellSingleQuote(buildChatCompletionsUrl(modelConfig.openaiBaseURL))} \\`,
+            `  -H ${shellSingleQuote(`Authorization: Bearer ${modelConfig.openaiApiKey || ''}`)} \\`,
+            `  -H ${shellSingleQuote('Content-Type: application/json')} \\`,
+            `  -d ${shellSingleQuote(JSON.stringify(payload, null, 2))}`
+        ].join('\n');
+    }
+    function buildCurlDedupKey(modelConfig) {
+        return JSON.stringify({
+            baseURL: modelConfig.openaiBaseURL || DEFAULT_OPENAI_BASE_URL,
+            apiKey: modelConfig.openaiApiKey || '',
+            modelName: modelConfig.modelName
+        });
+    }
+    function buildModelVerifyCurlCommands(configs) {
+        const commandMap = new Map();
+        for (const item of configs){
+            const key = buildCurlDedupKey(item.modelConfig);
+            const existing = commandMap.get(key);
+            if (existing) {
+                existing.intents.push(item.intent);
+                continue;
+            }
+            commandMap.set(key, {
+                intents: [
+                    item.intent
+                ],
+                curl: buildCurlCommand(item.modelConfig),
+                usesDefaultBaseURL: !item.modelConfig.openaiBaseURL
+            });
+        }
+        return [
+            ...commandMap.values()
+        ];
+    }
+    function formatModelVerifyFailureOutput(message, curlCommands) {
+        const details = message?.trim() || 'No failure details were generated.';
+        const curlSection = curlCommands.map((item)=>{
+            const baseUrlNote = item.usesDefaultBaseURL ? ' (base URL not configured; using OpenAI SDK default)' : '';
+            return external_chalk_default().gray(`# ${item.intents.join(', ')}${baseUrlNote}\n${item.curl}`);
+        }).join('\n\n');
+        return [
+            external_chalk_default().red.bold('❌ Model verify failed with messages:'),
+            MODEL_VERIFY_SEPARATOR,
+            '',
+            details,
+            '',
+            MODEL_VERIFY_SEPARATOR,
+            'Generated curl requests for basic API connectivity:',
+            'If the error is a basic connectivity issue, use these requests to test the base URL, API key, and model name directly.',
+            'These commands contain your API key. Do not share them publicly.',
+            '',
+            curlSection
+        ].join('\n');
+    }
+    function getDefaultDeps(io) {
+        return {
+            loadDotenv: ()=>{
+                loadDotenvConfig({
+                    dotenvDebug: true,
+                    dotenvOverride: true,
+                    log: io.stdout
+                });
+            },
+            getModelConfig: (intent)=>env_.globalModelConfigManager.getModelConfig(intent),
+            verifyModel: core_.runConnectivityTest
+        };
+    }
+    async function runModelVerifyCommand(args, deps, io, commandName) {
+        try {
+            if (args.includes('--help') || args.includes('-h')) {
+                io.stdout(modelCommandUsage(commandName));
+                return 0;
+            }
+            assertNoModelVerifyOptions(args, commandName);
+            io.stdout('Model verify started. This usually completes within 30 seconds.');
+            deps.loadDotenv();
+            io.stdout('');
+            const defaultModelConfig = deps.getModelConfig('default');
+            const planningModelConfig = deps.getModelConfig('planning');
+            const insightModelConfig = deps.getModelConfig('insight');
+            const curlCommands = buildModelVerifyCurlCommands([
+                {
+                    intent: 'default',
+                    modelConfig: defaultModelConfig
+                },
+                {
+                    intent: 'planning',
+                    modelConfig: planningModelConfig
+                },
+                {
+                    intent: 'insight',
+                    modelConfig: insightModelConfig
+                }
+            ]);
+            const baseURLs = new Map();
+            const modelRows = [
+                [
+                    'planning',
+                    planningModelConfig,
+                    'Text response'
+                ],
+                [
+                    'insight',
+                    insightModelConfig,
+                    'Image understanding'
+                ],
+                [
+                    'default',
+                    defaultModelConfig,
+                    'AI locate'
+                ]
+            ].map(([intent, config, check])=>{
+                const baseURL = config.openaiExtraConfig?.baseURL ?? config.openaiBaseURL;
+                const endpoint = baseURL ? displayEndpoint(String(baseURL)) : 'provider default';
+                if (!baseURLs.has(endpoint)) baseURLs.set(endpoint, baseURLs.size + 1);
+                return [
+                    intent,
+                    config.modelName,
+                    check,
+                    `[${baseURLs.get(endpoint)}]`
+                ];
+            });
+            io.stdout(formatTable([
+                'Model role',
+                'Model',
+                'Check',
+                'Endpoint'
+            ], modelRows));
+            io.stdout('');
+            for (const [endpoint, index] of baseURLs)io.stdout(`[${index}] Base URL: ${endpoint}`);
+            io.stdout('\nRunning 3 checks...\n');
+            const result = await deps.verifyModel({
+                defaultModelConfig,
+                planningModelConfig,
+                insightModelConfig
+            });
+            const checkLabels = {
+                text: 'Text response',
+                vision: 'Image understanding',
+                aiLocate: 'AI locate'
+            };
+            io.stdout(formatTable([
+                'Model role',
+                'Check',
+                'Result',
+                'Requests',
+                'Time'
+            ], result.checks.map((check)=>[
+                    check.intent,
+                    checkLabels[check.name],
+                    check.passed ? 'PASS' : 'FAIL',
+                    String(check.requestCount),
+                    `${check.durationMs} ms`
+                ])));
+            const requestEndpoints = new Map();
+            for (const check of result.checks){
+                for (const endpoint of check.endpoints){
+                    const display = displayEndpoint(endpoint);
+                    if (!requestEndpoints.has(display)) requestEndpoints.set(display, new Set());
+                    requestEndpoints.get(display).add(check.intent);
+                }
+                if (!check.passed && check.message) io.stdout(`\n${check.intent}: ${check.message}`);
+            }
+            if (requestEndpoints.size > 0) {
+                io.stdout('\nRequest endpoints:');
+                for (const [endpoint, intents] of requestEndpoints)io.stdout(`  ${[
+                    ...intents
+                ].join(', ')}: ${endpoint}`);
+            }
+            io.stdout('');
+            const passedChecks = result.checks.filter((check)=>check.passed).length;
+            io.stdout(`Requests sent: ${result.requestCount}. Checks: ${passedChecks} passed, ${result.checks.length - passedChecks} failed.`);
+            if (result.passed) {
+                io.stdout('✅ Model verify passed.');
+                return 0;
+            }
+            io.stderr(formatModelVerifyFailureOutput(result.message, curlCommands));
+            return 1;
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            io.stderr([
+                MODEL_VERIFY_SEPARATOR,
+                '❌ Model verify failed with messages:',
+                '',
+                message,
+                MODEL_VERIFY_SEPARATOR
+            ].join('\n'));
+            return 1;
+        }
+    }
+    async function runModelCommand(rawArgs, deps, io = {
+        stdout: console.log,
+        stderr: console.error
+    }, commandName = 'midscene-test') {
+        const [, action, ...restArgs] = rawArgs;
+        const mergedDeps = {
+            ...getDefaultDeps(io),
+            ...deps
+        };
+        if (!action || '--help' === action || '-h' === action) {
+            io.stdout(modelCommandUsage(commandName));
+            return 0;
+        }
+        if ('verify' === action) return runModelVerifyCommand(restArgs, mergedDeps, io, commandName);
+        if ('eval' === action) {
+            io.stderr(`${commandName} model eval is not implemented yet. It is reserved for future model evaluation suites.`);
+            return 1;
+        }
+        io.stderr(`Unknown ${commandName} model command: ${action}\n\n${modelCommandUsage(commandName)}`);
+        return 1;
+    }
+})();
+exports.buildModelVerifyCurlCommands = __webpack_exports__.buildModelVerifyCurlCommands;
+exports.loadDotenvConfig = __webpack_exports__.loadDotenvConfig;
+exports.runModelCommand = __webpack_exports__.runModelCommand;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "buildModelVerifyCurlCommands",
+    "loadDotenvConfig",
+    "runModelCommand"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=model-command.js.map

package/dist/lib/cli/model-command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli/model-command.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/cli/dotenv-loader.ts","../../../src/cli/model-command.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport dotenv from 'dotenv';\n\nexport interface DotenvLoadOptions {\n  dotenvOverride?: boolean;\n  dotenvDebug?: boolean;\n  cwd?: string;\n  log?: (message: string) => void;\n}\n\nexport function loadDotenvConfig(options: DotenvLoadOptions = {}) {\n  const dotEnvConfigFile = join(options.cwd ?? process.cwd(), '.env');\n  if (!existsSync(dotEnvConfigFile)) {\n    return;\n  }\n\n  options.log?.(`   Env file: ${dotEnvConfigFile}`);\n  dotenv.config({\n    path: dotEnvConfigFile,\n    debug: options.dotenvDebug,\n    override: options.dotenvOverride,\n  });\n}\n","import { runConnectivityTest } from '@midscene/core';\nimport {\n  type IModelConfig,\n  type TIntent,\n  globalModelConfigManager,\n} from '@midscene/shared/env';\nimport chalk from 'chalk';\nimport { loadDotenvConfig } from './dotenv-loader';\n\nconst DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1';\nconst MODEL_VERIFY_SEPARATOR = '────────────────────────────────────────';\nconst modelCommandUsage = (commandName: string) => `Usage:\n  ${commandName} model verify\n\nVerify model connectivity and Midscene compatibility for default, planning, and insight models.\nReads .env from the current working directory, overriding shell environment variables.\nShows endpoints, model names, request counts, and per-check results with duration.\nReturns exit code 0 on success and 1 on failure.\n\nOptions:\n  --help, -h  Show this help\n`;\n\ninterface ModelCommandIO {\n  stdout: (message: string) => void;\n  stderr: (message: string) => void;\n}\n\ninterface ModelCommandDeps {\n  loadDotenv: () => void;\n  getModelConfig: (intent: TIntent) => IModelConfig;\n  verifyModel: typeof runConnectivityTest;\n}\n\nexport interface CurlCommandItem {\n  intents: TIntent[];\n  curl: string;\n  usesDefaultBaseURL: boolean;\n}\n\nfunction assertNoModelVerifyOptions(args: string[], commandName: string) {\n  for (const arg of args) {\n    throw new Error(`Unknown option for ${commandName} model verify: ${arg}`);\n  }\n}\n\nfunction formatTable(headers: string[], rows: string[][]): string {\n  const widths = headers.map((header, index) =>\n    Math.max(header.length, ...rows.map((row) => row[index].length)),\n  );\n  const formatRow = (row: string[]) =>\n    row\n      .map((cell, index) => cell.padEnd(widths[index]))\n      .join('  ')\n      .trimEnd();\n  return [formatRow(headers), ...rows.map(formatRow)].join('\\n');\n}\n\nfunction displayEndpoint(endpoint: string): string {\n  try {\n    const url = new URL(endpoint);\n    url.username = '';\n    url.password = '';\n    url.search = '';\n    url.hash = '';\n    return url.toString();\n  } catch {\n    return '(invalid endpoint URL)';\n  }\n}\n\nfunction shellSingleQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`;\n}\n\nfunction buildChatCompletionsUrl(baseURL?: string): string {\n  const normalizedBaseURL = (baseURL || DEFAULT_OPENAI_BASE_URL).replace(\n    /\\/+$/,\n    '',\n  );\n  if (normalizedBaseURL.endsWith('/chat/completions')) {\n    return normalizedBaseURL;\n  }\n  return `${normalizedBaseURL}/chat/completions`;\n}\n\nfunction buildCurlCommand(modelConfig: IModelConfig): string {\n  const payload = {\n    model: modelConfig.modelName,\n    messages: [{ role: 'user', content: 'What is 1+1?' }],\n  };\n\n  return [\n    `curl -X POST ${shellSingleQuote(buildChatCompletionsUrl(modelConfig.openaiBaseURL))} \\\\`,\n    `  -H ${shellSingleQuote(`Authorization: Bearer ${modelConfig.openaiApiKey || ''}`)} \\\\`,\n    `  -H ${shellSingleQuote('Content-Type: application/json')} \\\\`,\n    `  -d ${shellSingleQuote(JSON.stringify(payload, null, 2))}`,\n  ].join('\\n');\n}\n\nfunction buildCurlDedupKey(modelConfig: IModelConfig): string {\n  return JSON.stringify({\n    baseURL: modelConfig.openaiBaseURL || DEFAULT_OPENAI_BASE_URL,\n    apiKey: modelConfig.openaiApiKey || '',\n    modelName: modelConfig.modelName,\n  });\n}\n\nexport function buildModelVerifyCurlCommands(\n  configs: Array<{ intent: TIntent; modelConfig: IModelConfig }>,\n): CurlCommandItem[] {\n  const commandMap = new Map<string, CurlCommandItem>();\n\n  for (const item of configs) {\n    const key = buildCurlDedupKey(item.modelConfig);\n    const existing = commandMap.get(key);\n    if (existing) {\n      existing.intents.push(item.intent);\n      continue;\n    }\n    commandMap.set(key, {\n      intents: [item.intent],\n      curl: buildCurlCommand(item.modelConfig),\n      usesDefaultBaseURL: !item.modelConfig.openaiBaseURL,\n    });\n  }\n\n  return [...commandMap.values()];\n}\n\nfunction formatModelVerifyFailureOutput(\n  message: string | undefined,\n  curlCommands: CurlCommandItem[],\n): string {\n  const details = message?.trim() || 'No failure details were generated.';\n  const curlSection = curlCommands\n    .map((item) => {\n      const baseUrlNote = item.usesDefaultBaseURL\n        ? ' (base URL not configured; using OpenAI SDK default)'\n        : '';\n      return chalk.gray(\n        `# ${item.intents.join(', ')}${baseUrlNote}\\n${item.curl}`,\n      );\n    })\n    .join('\\n\\n');\n\n  return [\n    chalk.red.bold('❌ Model verify failed with messages:'),\n    MODEL_VERIFY_SEPARATOR,\n    '',\n    details,\n    '',\n    MODEL_VERIFY_SEPARATOR,\n    'Generated curl requests for basic API connectivity:',\n    'If the error is a basic connectivity issue, use these requests to test the base URL, API key, and model name directly.',\n    'These commands contain your API key. Do not share them publicly.',\n    '',\n    curlSection,\n  ].join('\\n');\n}\n\nfunction getDefaultDeps(io: ModelCommandIO): ModelCommandDeps {\n  return {\n    loadDotenv: () => {\n      loadDotenvConfig({\n        dotenvDebug: true,\n        dotenvOverride: true,\n        log: io.stdout,\n      });\n    },\n    getModelConfig: (intent) => globalModelConfigManager.getModelConfig(intent),\n    verifyModel: runConnectivityTest,\n  };\n}\n\nasync function runModelVerifyCommand(\n  args: string[],\n  deps: ModelCommandDeps,\n  io: ModelCommandIO,\n  commandName: string,\n): Promise<number> {\n  try {\n    if (args.includes('--help') || args.includes('-h')) {\n      io.stdout(modelCommandUsage(commandName));\n      return 0;\n    }\n\n    assertNoModelVerifyOptions(args, commandName);\n    io.stdout(\n      'Model verify started. This usually completes within 30 seconds.',\n    );\n    deps.loadDotenv();\n    io.stdout('');\n\n    const defaultModelConfig = deps.getModelConfig('default');\n    const planningModelConfig = deps.getModelConfig('planning');\n    const insightModelConfig = deps.getModelConfig('insight');\n    const curlCommands = buildModelVerifyCurlCommands([\n      { intent: 'default', modelConfig: defaultModelConfig },\n      { intent: 'planning', modelConfig: planningModelConfig },\n      { intent: 'insight', modelConfig: insightModelConfig },\n    ]);\n\n    const baseURLs = new Map<string, number>();\n    const modelRows = (\n      [\n        ['planning', planningModelConfig, 'Text response'],\n        ['insight', insightModelConfig, 'Image understanding'],\n        ['default', defaultModelConfig, 'AI locate'],\n      ] as const\n    ).map(([intent, config, check]) => {\n      const baseURL = config.openaiExtraConfig?.baseURL ?? config.openaiBaseURL;\n      const endpoint = baseURL\n        ? displayEndpoint(String(baseURL))\n        : 'provider default';\n      if (!baseURLs.has(endpoint)) baseURLs.set(endpoint, baseURLs.size + 1);\n      return [intent, config.modelName, check, `[${baseURLs.get(endpoint)}]`];\n    });\n    io.stdout(\n      formatTable(['Model role', 'Model', 'Check', 'Endpoint'], modelRows),\n    );\n    io.stdout('');\n    for (const [endpoint, index] of baseURLs) {\n      io.stdout(`[${index}] Base URL: ${endpoint}`);\n    }\n    io.stdout('\\nRunning 3 checks...\\n');\n\n    const result = await deps.verifyModel({\n      defaultModelConfig,\n      planningModelConfig,\n      insightModelConfig,\n    });\n\n    const checkLabels = {\n      text: 'Text response',\n      vision: 'Image understanding',\n      aiLocate: 'AI locate',\n    };\n    io.stdout(\n      formatTable(\n        ['Model role', 'Check', 'Result', 'Requests', 'Time'],\n        result.checks.map((check) => [\n          check.intent,\n          checkLabels[check.name],\n          check.passed ? 'PASS' : 'FAIL',\n          String(check.requestCount),\n          `${check.durationMs} ms`,\n        ]),\n      ),\n    );\n    const requestEndpoints = new Map<string, Set<string>>();\n    for (const check of result.checks) {\n      for (const endpoint of check.endpoints) {\n        const display = displayEndpoint(endpoint);\n        if (!requestEndpoints.has(display))\n          requestEndpoints.set(display, new Set());\n        requestEndpoints.get(display)!.add(check.intent);\n      }\n      if (!check.passed && check.message)\n        io.stdout(`\\n${check.intent}: ${check.message}`);\n    }\n    if (requestEndpoints.size > 0) {\n      io.stdout('\\nRequest endpoints:');\n      for (const [endpoint, intents] of requestEndpoints) {\n        io.stdout(`  ${[...intents].join(', ')}: ${endpoint}`);\n      }\n    }\n    io.stdout('');\n    const passedChecks = result.checks.filter((check) => check.passed).length;\n    io.stdout(\n      `Requests sent: ${result.requestCount}. Checks: ${passedChecks} passed, ${result.checks.length - passedChecks} failed.`,\n    );\n\n    if (result.passed) {\n      io.stdout('✅ Model verify passed.');\n      return 0;\n    }\n\n    io.stderr(formatModelVerifyFailureOutput(result.message, curlCommands));\n    return 1;\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    io.stderr(\n      [\n        MODEL_VERIFY_SEPARATOR,\n        '❌ Model verify failed with messages:',\n        '',\n        message,\n        MODEL_VERIFY_SEPARATOR,\n      ].join('\\n'),\n    );\n    return 1;\n  }\n}\n\nexport async function runModelCommand(\n  rawArgs: string[],\n  deps?: Partial<ModelCommandDeps>,\n  io: ModelCommandIO = {\n    stdout: console.log,\n    stderr: console.error,\n  },\n  commandName = 'midscene-test',\n): Promise<number> {\n  const [, action, ...restArgs] = rawArgs;\n  const mergedDeps = {\n    ...getDefaultDeps(io),\n    ...deps,\n  };\n\n  if (!action || action === '--help' || action === '-h') {\n    io.stdout(modelCommandUsage(commandName));\n    return 0;\n  }\n\n  if (action === 'verify') {\n    return runModelVerifyCommand(restArgs, mergedDeps, io, commandName);\n  }\n\n  if (action === 'eval') {\n    io.stderr(\n      `${commandName} model eval is not implemented yet. It is reserved for future model evaluation suites.`,\n    );\n    return 1;\n  }\n\n  io.stderr(\n    `Unknown ${commandName} model command: ${action}\\n\\n${modelCommandUsage(commandName)}`,\n  );\n  return 1;\n}\n\nexport { loadDotenvConfig, type DotenvLoadOptions } from './dotenv-loader';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","loadDotenvConfig","options","dotEnvConfigFile","join","process","existsSync","dotenv","DEFAULT_OPENAI_BASE_URL","MODEL_VERIFY_SEPARATOR","modelCommandUsage","commandName","assertNoModelVerifyOptions","args","arg","Error","formatTable","headers","rows","widths","header","index","Math","row","formatRow","cell","displayEndpoint","endpoint","url","URL","shellSingleQuote","value","buildChatCompletionsUrl","baseURL","normalizedBaseURL","buildCurlCommand","modelConfig","payload","JSON","buildCurlDedupKey","buildModelVerifyCurlCommands","configs","commandMap","Map","item","existing","formatModelVerifyFailureOutput","message","curlCommands","details","curlSection","baseUrlNote","chalk","getDefaultDeps","io","intent","globalModelConfigManager","runConnectivityTest","runModelVerifyCommand","deps","defaultModelConfig","planningModelConfig","insightModelConfig","baseURLs","modelRows","config","check","String","result","checkLabels","requestEndpoints","display","Set","intents","passedChecks","error","runModelCommand","rawArgs","console","action","restArgs","mergedDeps"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ICKO,SAASI,iBAAiBC,UAA6B,CAAC,CAAC;QAC9D,MAAMC,mBAAmBC,AAAAA,IAAAA,oBAAAA,IAAAA,AAAAA,EAAKF,QAAQ,GAAG,IAAIG,QAAQ,GAAG,IAAI;QAC5D,IAAI,CAACC,AAAAA,IAAAA,kBAAAA,UAAAA,AAAAA,EAAWH,mBACd;QAGFD,QAAQ,GAAG,GAAG,CAAC,aAAa,EAAEC,kBAAkB;QAChDI,0BAAAA,MAAa,CAAC;YACZ,MAAMJ;YACN,OAAOD,QAAQ,WAAW;YAC1B,UAAUA,QAAQ,cAAc;QAClC;IACF;ICdA,MAAMM,0BAA0B;IAChC,MAAMC,yBAAyB;IAC/B,MAAMC,oBAAoB,CAACC,cAAwB,CAAC;EAClD,EAAEA,YAAY;;;;;;;;;AAShB,CAAC;IAmBD,SAASC,2BAA2BC,IAAc,EAAEF,WAAmB;QACrE,KAAK,MAAMG,OAAOD,KAChB,MAAM,IAAIE,MAAM,CAAC,mBAAmB,EAAEJ,YAAY,eAAe,EAAEG,KAAK;IAE5E;IAEA,SAASE,YAAYC,OAAiB,EAAEC,IAAgB;QACtD,MAAMC,SAASF,QAAQ,GAAG,CAAC,CAACG,QAAQC,QAClCC,KAAK,GAAG,CAACF,OAAO,MAAM,KAAKF,KAAK,GAAG,CAAC,CAACK,MAAQA,GAAG,CAACF,MAAM,CAAC,MAAM;QAEhE,MAAMG,YAAY,CAACD,MACjBA,IACG,GAAG,CAAC,CAACE,MAAMJ,QAAUI,KAAK,MAAM,CAACN,MAAM,CAACE,MAAM,GAC9C,IAAI,CAAC,MACL,OAAO;QACZ,OAAO;YAACG,UAAUP;eAAaC,KAAK,GAAG,CAACM;SAAW,CAAC,IAAI,CAAC;IAC3D;IAEA,SAASE,gBAAgBC,QAAgB;QACvC,IAAI;YACF,MAAMC,MAAM,IAAIC,IAAIF;YACpBC,IAAI,QAAQ,GAAG;YACfA,IAAI,QAAQ,GAAG;YACfA,IAAI,MAAM,GAAG;YACbA,IAAI,IAAI,GAAG;YACX,OAAOA,IAAI,QAAQ;QACrB,EAAE,OAAM;YACN,OAAO;QACT;IACF;IAEA,SAASE,iBAAiBC,KAAa;QACrC,OAAO,CAAC,CAAC,EAAEA,MAAM,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;IAC5C;IAEA,SAASC,wBAAwBC,OAAgB;QAC/C,MAAMC,oBAAqBD,AAAAA,CAAAA,WAAWzB,uBAAsB,EAAG,OAAO,CACpE,QACA;QAEF,IAAI0B,kBAAkB,QAAQ,CAAC,sBAC7B,OAAOA;QAET,OAAO,GAAGA,kBAAkB,iBAAiB,CAAC;IAChD;IAEA,SAASC,iBAAiBC,WAAyB;QACjD,MAAMC,UAAU;YACd,OAAOD,YAAY,SAAS;YAC5B,UAAU;gBAAC;oBAAE,MAAM;oBAAQ,SAAS;gBAAe;aAAE;QACvD;QAEA,OAAO;YACL,CAAC,aAAa,EAAEN,iBAAiBE,wBAAwBI,YAAY,aAAa,GAAG,GAAG,CAAC;YACzF,CAAC,KAAK,EAAEN,iBAAiB,CAAC,sBAAsB,EAAEM,YAAY,YAAY,IAAI,IAAI,EAAE,GAAG,CAAC;YACxF,CAAC,KAAK,EAAEN,iBAAiB,kCAAkC,GAAG,CAAC;YAC/D,CAAC,KAAK,EAAEA,iBAAiBQ,KAAK,SAAS,CAACD,SAAS,MAAM,KAAK;SAC7D,CAAC,IAAI,CAAC;IACT;IAEA,SAASE,kBAAkBH,WAAyB;QAClD,OAAOE,KAAK,SAAS,CAAC;YACpB,SAASF,YAAY,aAAa,IAAI5B;YACtC,QAAQ4B,YAAY,YAAY,IAAI;YACpC,WAAWA,YAAY,SAAS;QAClC;IACF;IAEO,SAASI,6BACdC,OAA8D;QAE9D,MAAMC,aAAa,IAAIC;QAEvB,KAAK,MAAMC,QAAQH,QAAS;YAC1B,MAAM7C,MAAM2C,kBAAkBK,KAAK,WAAW;YAC9C,MAAMC,WAAWH,WAAW,GAAG,CAAC9C;YAChC,IAAIiD,UAAU;gBACZA,SAAS,OAAO,CAAC,IAAI,CAACD,KAAK,MAAM;gBACjC;YACF;YACAF,WAAW,GAAG,CAAC9C,KAAK;gBAClB,SAAS;oBAACgD,KAAK,MAAM;iBAAC;gBACtB,MAAMT,iBAAiBS,KAAK,WAAW;gBACvC,oBAAoB,CAACA,KAAK,WAAW,CAAC,aAAa;YACrD;QACF;QAEA,OAAO;eAAIF,WAAW,MAAM;SAAG;IACjC;IAEA,SAASI,+BACPC,OAA2B,EAC3BC,YAA+B;QAE/B,MAAMC,UAAUF,SAAS,UAAU;QACnC,MAAMG,cAAcF,aACjB,GAAG,CAAC,CAACJ;YACJ,MAAMO,cAAcP,KAAK,kBAAkB,GACvC,yDACA;YACJ,OAAOQ,yBAAAA,IAAU,CACf,CAAC,EAAE,EAAER,KAAK,OAAO,CAAC,IAAI,CAAC,QAAQO,YAAY,EAAE,EAAEP,KAAK,IAAI,EAAE;QAE9D,GACC,IAAI,CAAC;QAER,OAAO;YACLQ,yBAAAA,GAAAA,CAAAA,IAAc,CAAC;YACf3C;YACA;YACAwC;YACA;YACAxC;YACA;YACA;YACA;YACA;YACAyC;SACD,CAAC,IAAI,CAAC;IACT;IAEA,SAASG,eAAeC,EAAkB;QACxC,OAAO;YACL,YAAY;gBACVrD,iBAAiB;oBACf,aAAa;oBACb,gBAAgB;oBAChB,KAAKqD,GAAG,MAAM;gBAChB;YACF;YACA,gBAAgB,CAACC,SAAWC,KAAAA,wBAAAA,CAAAA,cAAuC,CAACD;YACpE,aAAaE,MAAAA,mBAAmBA;QAClC;IACF;IAEA,eAAeC,sBACb7C,IAAc,EACd8C,IAAsB,EACtBL,EAAkB,EAClB3C,WAAmB;QAEnB,IAAI;YACF,IAAIE,KAAK,QAAQ,CAAC,aAAaA,KAAK,QAAQ,CAAC,OAAO;gBAClDyC,GAAG,MAAM,CAAC5C,kBAAkBC;gBAC5B,OAAO;YACT;YAEAC,2BAA2BC,MAAMF;YACjC2C,GAAG,MAAM,CACP;YAEFK,KAAK,UAAU;YACfL,GAAG,MAAM,CAAC;YAEV,MAAMM,qBAAqBD,KAAK,cAAc,CAAC;YAC/C,MAAME,sBAAsBF,KAAK,cAAc,CAAC;YAChD,MAAMG,qBAAqBH,KAAK,cAAc,CAAC;YAC/C,MAAMX,eAAeR,6BAA6B;gBAChD;oBAAE,QAAQ;oBAAW,aAAaoB;gBAAmB;gBACrD;oBAAE,QAAQ;oBAAY,aAAaC;gBAAoB;gBACvD;oBAAE,QAAQ;oBAAW,aAAaC;gBAAmB;aACtD;YAED,MAAMC,WAAW,IAAIpB;YACrB,MAAMqB,YACJ;gBACE;oBAAC;oBAAYH;oBAAqB;iBAAgB;gBAClD;oBAAC;oBAAWC;oBAAoB;iBAAsB;gBACtD;oBAAC;oBAAWF;oBAAoB;iBAAY;aAC7C,CACD,GAAG,CAAC,CAAC,CAACL,QAAQU,QAAQC,MAAM;gBAC5B,MAAMjC,UAAUgC,OAAO,iBAAiB,EAAE,WAAWA,OAAO,aAAa;gBACzE,MAAMtC,WAAWM,UACbP,gBAAgByC,OAAOlC,YACvB;gBACJ,IAAI,CAAC8B,SAAS,GAAG,CAACpC,WAAWoC,SAAS,GAAG,CAACpC,UAAUoC,SAAS,IAAI,GAAG;gBACpE,OAAO;oBAACR;oBAAQU,OAAO,SAAS;oBAAEC;oBAAO,CAAC,CAAC,EAAEH,SAAS,GAAG,CAACpC,UAAU,CAAC,CAAC;iBAAC;YACzE;YACA2B,GAAG,MAAM,CACPtC,YAAY;gBAAC;gBAAc;gBAAS;gBAAS;aAAW,EAAEgD;YAE5DV,GAAG,MAAM,CAAC;YACV,KAAK,MAAM,CAAC3B,UAAUN,MAAM,IAAI0C,SAC9BT,GAAG,MAAM,CAAC,CAAC,CAAC,EAAEjC,MAAM,YAAY,EAAEM,UAAU;YAE9C2B,GAAG,MAAM,CAAC;YAEV,MAAMc,SAAS,MAAMT,KAAK,WAAW,CAAC;gBACpCC;gBACAC;gBACAC;YACF;YAEA,MAAMO,cAAc;gBAClB,MAAM;gBACN,QAAQ;gBACR,UAAU;YACZ;YACAf,GAAG,MAAM,CACPtC,YACE;gBAAC;gBAAc;gBAAS;gBAAU;gBAAY;aAAO,EACrDoD,OAAO,MAAM,CAAC,GAAG,CAAC,CAACF,QAAU;oBAC3BA,MAAM,MAAM;oBACZG,WAAW,CAACH,MAAM,IAAI,CAAC;oBACvBA,MAAM,MAAM,GAAG,SAAS;oBACxBC,OAAOD,MAAM,YAAY;oBACzB,GAAGA,MAAM,UAAU,CAAC,GAAG,CAAC;iBACzB;YAGL,MAAMI,mBAAmB,IAAI3B;YAC7B,KAAK,MAAMuB,SAASE,OAAO,MAAM,CAAE;gBACjC,KAAK,MAAMzC,YAAYuC,MAAM,SAAS,CAAE;oBACtC,MAAMK,UAAU7C,gBAAgBC;oBAChC,IAAI,CAAC2C,iBAAiB,GAAG,CAACC,UACxBD,iBAAiB,GAAG,CAACC,SAAS,IAAIC;oBACpCF,iBAAiB,GAAG,CAACC,SAAU,GAAG,CAACL,MAAM,MAAM;gBACjD;gBACA,IAAI,CAACA,MAAM,MAAM,IAAIA,MAAM,OAAO,EAChCZ,GAAG,MAAM,CAAC,CAAC,EAAE,EAAEY,MAAM,MAAM,CAAC,EAAE,EAAEA,MAAM,OAAO,EAAE;YACnD;YACA,IAAII,iBAAiB,IAAI,GAAG,GAAG;gBAC7BhB,GAAG,MAAM,CAAC;gBACV,KAAK,MAAM,CAAC3B,UAAU8C,QAAQ,IAAIH,iBAChChB,GAAG,MAAM,CAAC,CAAC,EAAE,EAAE;uBAAImB;iBAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE9C,UAAU;YAEzD;YACA2B,GAAG,MAAM,CAAC;YACV,MAAMoB,eAAeN,OAAO,MAAM,CAAC,MAAM,CAAC,CAACF,QAAUA,MAAM,MAAM,EAAE,MAAM;YACzEZ,GAAG,MAAM,CACP,CAAC,eAAe,EAAEc,OAAO,YAAY,CAAC,UAAU,EAAEM,aAAa,SAAS,EAAEN,OAAO,MAAM,CAAC,MAAM,GAAGM,aAAa,QAAQ,CAAC;YAGzH,IAAIN,OAAO,MAAM,EAAE;gBACjBd,GAAG,MAAM,CAAC;gBACV,OAAO;YACT;YAEAA,GAAG,MAAM,CAACR,+BAA+BsB,OAAO,OAAO,EAAEpB;YACzD,OAAO;QACT,EAAE,OAAO2B,OAAO;YACd,MAAM5B,UAAU4B,iBAAiB5D,QAAQ4D,MAAM,OAAO,GAAGR,OAAOQ;YAChErB,GAAG,MAAM,CACP;gBACE7C;gBACA;gBACA;gBACAsC;gBACAtC;aACD,CAAC,IAAI,CAAC;YAET,OAAO;QACT;IACF;IAEO,eAAemE,gBACpBC,OAAiB,EACjBlB,IAAgC,EAChCL,KAAqB;QACnB,QAAQwB,QAAQ,GAAG;QACnB,QAAQA,QAAQ,KAAK;IACvB,CAAC,EACDnE,cAAc,eAAe;QAE7B,MAAM,GAAGoE,QAAQ,GAAGC,SAAS,GAAGH;QAChC,MAAMI,aAAa;YACjB,GAAG5B,eAAeC,GAAG;YACrB,GAAGK,IAAI;QACT;QAEA,IAAI,CAACoB,UAAUA,AAAW,aAAXA,UAAuBA,AAAW,SAAXA,QAAiB;YACrDzB,GAAG,MAAM,CAAC5C,kBAAkBC;YAC5B,OAAO;QACT;QAEA,IAAIoE,AAAW,aAAXA,QACF,OAAOrB,sBAAsBsB,UAAUC,YAAY3B,IAAI3C;QAGzD,IAAIoE,AAAW,WAAXA,QAAmB;YACrBzB,GAAG,MAAM,CACP,GAAG3C,YAAY,sFAAsF,CAAC;YAExG,OAAO;QACT;QAEA2C,GAAG,MAAM,CACP,CAAC,QAAQ,EAAE3C,YAAY,gBAAgB,EAAEoE,OAAO,IAAI,EAAErE,kBAAkBC,cAAc;QAExF,OAAO;IACT"}

package/dist/lib/test-cli.js

@@ -3,6 +3,18 @@ var __webpack_modules__ = {
     "@inquirer/prompts" (module) {
         module.exports = require("@inquirer/prompts");
     },
+    "@midscene/core" (module) {
+        module.exports = require("@midscene/core");
+    },
+    "@midscene/shared/env" (module) {
+        module.exports = require("@midscene/shared/env");
+    },
+    chalk (module) {
+        module.exports = require("chalk");
+    },
+    dotenv (module) {
+        module.exports = require("dotenv");
+    },
     execa (module) {
         module.exports = require("execa");
     },
@@ -17,7 +29,7 @@ var __webpack_modules__ = {
     },
     "./package.json" (module) {
         module.exports = {
-            rE: "1.14.0"
+            rE: "1.15.0"
         };
     }
 };
@@ -80,7 +92,14 @@ var __webpack_exports__ = {};
 (()=>{
     var external_node_fs_ = __webpack_require__("node:fs");
     var external_node_path_ = __webpack_require__("node:path");
+    const test_runner_namespaceObject = require("@midscene/core/internal/test-runner");
     var package_0 = __webpack_require__("./package.json");
+    const external_node_util_namespaceObject = require("node:util");
+    const formatCliError = (error)=>'string' == typeof error ? error : (0, external_node_util_namespaceObject.inspect)(error, {
+            colors: false,
+            depth: null,
+            customInspect: false
+        });
     const v4_namespaceObject = require("zod/v4");
     const GENERATED_COMMENT = '<!-- Generated by `midscene-test nodes`. Do not edit directly. -->';
     const sortNodesForSpec = (nodes)=>{
@@ -217,7 +236,6 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
     const external_node_url_namespaceObject = require("node:url");
     const api_namespaceObject = require("tsx/cjs/api");
     const esm_api_namespaceObject = require("tsx/esm/api");
-    const test_runner_namespaceObject = require("@midscene/core/internal/test-runner");
     const NodeRegistry = test_runner_namespaceObject.NodeRegistry;
     const projectIdFromIndex = (index)=>`project-${index}`;
     const getDefaultExport = (loaded, absolutePath)=>{
@@ -1576,6 +1594,7 @@ Usage:
   midscene-test --config <midscene.config.ts> [options]
   midscene-test nodes [directory] [--config midscene.config.ts]
   midscene-test create --help
+  midscene-test model verify    Check model connectivity and Midscene compatibility
 
 Options:
   --config <path>              Native TypeScript or JavaScript Test config
@@ -1641,8 +1660,78 @@ Legacy tasks/flow YAML and its CLI options belong to the midscene command.
             io.log(`\nNode Spec generated: ${specPath}`);
         }
     };
+    const reportTestProjectResult = (result, io, infrastructureErrors = result.errors ?? [])=>{
+        const reportedErrors = new Set();
+        const reportError = (source, error)=>{
+            if (error instanceof Error && reportedErrors.has(error)) return;
+            io.error(`midscene-test: ${source}: ${formatCliError(error)}`);
+            const pending = [
+                error
+            ];
+            while(pending.length > 0){
+                const current = pending.pop();
+                if (!(current instanceof Error) || reportedErrors.has(current)) continue;
+                reportedErrors.add(current);
+                pending.push(current.cause);
+                if (current instanceof AggregateError) pending.push(...current.errors);
+            }
+        };
+        for (const failure of result.collectionErrors)reportError(`${failure.projectName}/${failure.sourcePath}`, failure.error);
+        const finalCases = new Map(result.cases.map((outcome)=>[
+                outcome.caseId,
+                outcome
+            ]));
+        for (const outcome of finalCases.values()){
+            if ('failed' !== outcome.status || !outcome.run) continue;
+            const run = outcome.run;
+            const source = `${run.projectName}/${run.sourcePath} / ${run.name}`;
+            for (const step of [
+                ...run.beforeEach,
+                ...run.steps,
+                ...run.afterEach
+            ])if (step.error) reportError(`${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}`, step.error);
+            for (const error of [
+                ...run.executionErrors ?? [],
+                ...run.teardownErrors ?? []
+            ])reportError(source, error);
+        }
+        const finalDocuments = new Map(result.documents.map((document)=>[
+                document.documentId,
+                document
+            ]));
+        for (const document of finalDocuments.values()){
+            if ('failed' !== document.status) continue;
+            const source = `${document.projectName}/${document.sourcePath}`;
+            for (const step of [
+                ...document.beforeAll,
+                ...document.afterAll
+            ])if (step.error) reportError(`${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}`, step.error);
+            for (const error of [
+                ...document.executionErrors ?? [],
+                ...document.teardownErrors ?? []
+            ])reportError(source, error);
+            for (const failure of document.hostErrors ?? [])reportError(`${source} / host ${failure.phase}`, failure.error);
+        }
+        for (const project of result.projects){
+            if (project.lifecycle?.setupError) reportError(`${project.name} / setup`, project.lifecycle.setupError);
+            for (const error of project.lifecycle?.teardownErrors ?? [])reportError(`${project.name} / teardown`, error);
+        }
+        for (const error of infrastructureErrors)reportError('infrastructure', error);
+        io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
+        io.log(`Results: ${result.resultDir}`);
+        io.log(`Summary: ${result.summaryPath}`);
+        if (result.reportPath) io.log(`Report: ${result.reportPath}`);
+        return result.exitCode;
+    };
     async function runTestCli(args, io = defaultCliIO) {
         try {
+            if ('model' === args[0]) {
+                const { runModelCommand } = await __webpack_require__.e("703").then(__webpack_require__.bind(__webpack_require__, "./src/cli/model-command.ts"));
+                return runModelCommand(args, void 0, {
+                    stdout: io.log,
+                    stderr: io.error
+                });
+            }
             if ('create' === args[0]) {
                 const { runCreateCommand } = await __webpack_require__.e("160").then(__webpack_require__.bind(__webpack_require__, "./src/cli/create-command.ts"));
                 await runCreateCommand(args.slice(1), io);
@@ -1661,36 +1750,20 @@ Legacy tasks/flow YAML and its CLI options belong to the midscene command.
                 await runNodesCommand(options, io);
                 return 0;
             }
-            const result = await runTestProject({
-                ...options,
-                onProgress: (message)=>io.log(message)
-            });
-            for (const failure of result.collectionErrors)io.error(`midscene-test: ${failure.projectName}/${failure.sourcePath}: ${failure.error.message}`);
-            const finalCases = new Map(result.cases.map((outcome)=>[
-                    outcome.caseId,
-                    outcome
-                ]));
-            for (const outcome of finalCases.values()){
-                if ('failed' !== outcome.status || !outcome.run) continue;
-                const run = outcome.run;
-                const source = `${run.projectName}/${run.sourcePath} / ${run.name}`;
-                for (const step of [
-                    ...run.beforeEach,
-                    ...run.steps,
-                    ...run.afterEach
-                ])if (step.error) io.error(`midscene-test: ${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}: ${step.error.message}`);
-                for (const error of [
-                    ...run.executionErrors ?? [],
-                    ...run.teardownErrors ?? []
-                ])io.error(`midscene-test: ${source}: ${error.message}`);
+            let result;
+            try {
+                result = await runTestProject({
+                    ...options,
+                    onProgress: (message)=>io.log(message)
+                });
+            } catch (error) {
+                if (!(error instanceof test_runner_namespaceObject.WorkflowExecutionFailure)) throw error;
+                reportTestProjectResult(error.result, io, error.errors);
+                return 1;
             }
-            io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
-            io.log(`Results: ${result.resultDir}`);
-            io.log(`Summary: ${result.summaryPath}`);
-            if (result.reportPath) io.log(`Report: ${result.reportPath}`);
-            return result.exitCode;
+            return reportTestProjectResult(result, io);
         } catch (error) {
-            io.error(error instanceof Error ? error.message : String(error));
+            io.error(formatCliError(error));
             return 1;
         }
     }