@midscene/test 1.14.0 → 1.15.0

package/dist/es/cli/model-command.mjs

@@ -0,0 +1,285 @@
+import { runConnectivityTest } from "@midscene/core";
+import { globalModelConfigManager } from "@midscene/shared/env";
+import chalk from "chalk";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import dotenv from "dotenv";
+function loadDotenvConfig(options = {}) {
+    const dotEnvConfigFile = join(options.cwd ?? process.cwd(), '.env');
+    if (!existsSync(dotEnvConfigFile)) return;
+    options.log?.(`   Env file: ${dotEnvConfigFile}`);
+    dotenv.config({
+        path: dotEnvConfigFile,
+        debug: options.dotenvDebug,
+        override: options.dotenvOverride
+    });
+}
+const DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1';
+const MODEL_VERIFY_SEPARATOR = '────────────────────────────────────────';
+const modelCommandUsage = (commandName)=>`Usage:
+  ${commandName} model verify
+
+Verify model connectivity and Midscene compatibility for default, planning, and insight models.
+Reads .env from the current working directory, overriding shell environment variables.
+Shows endpoints, model names, request counts, and per-check results with duration.
+Returns exit code 0 on success and 1 on failure.
+
+Options:
+  --help, -h  Show this help
+`;
+function assertNoModelVerifyOptions(args, commandName) {
+    for (const arg of args)throw new Error(`Unknown option for ${commandName} model verify: ${arg}`);
+}
+function formatTable(headers, rows) {
+    const widths = headers.map((header, index)=>Math.max(header.length, ...rows.map((row)=>row[index].length)));
+    const formatRow = (row)=>row.map((cell, index)=>cell.padEnd(widths[index])).join('  ').trimEnd();
+    return [
+        formatRow(headers),
+        ...rows.map(formatRow)
+    ].join('\n');
+}
+function displayEndpoint(endpoint) {
+    try {
+        const url = new URL(endpoint);
+        url.username = '';
+        url.password = '';
+        url.search = '';
+        url.hash = '';
+        return url.toString();
+    } catch  {
+        return '(invalid endpoint URL)';
+    }
+}
+function shellSingleQuote(value) {
+    return `'${value.replace(/'/g, "'\\''")}'`;
+}
+function buildChatCompletionsUrl(baseURL) {
+    const normalizedBaseURL = (baseURL || DEFAULT_OPENAI_BASE_URL).replace(/\/+$/, '');
+    if (normalizedBaseURL.endsWith('/chat/completions')) return normalizedBaseURL;
+    return `${normalizedBaseURL}/chat/completions`;
+}
+function buildCurlCommand(modelConfig) {
+    const payload = {
+        model: modelConfig.modelName,
+        messages: [
+            {
+                role: 'user',
+                content: 'What is 1+1?'
+            }
+        ]
+    };
+    return [
+        `curl -X POST ${shellSingleQuote(buildChatCompletionsUrl(modelConfig.openaiBaseURL))} \\`,
+        `  -H ${shellSingleQuote(`Authorization: Bearer ${modelConfig.openaiApiKey || ''}`)} \\`,
+        `  -H ${shellSingleQuote('Content-Type: application/json')} \\`,
+        `  -d ${shellSingleQuote(JSON.stringify(payload, null, 2))}`
+    ].join('\n');
+}
+function buildCurlDedupKey(modelConfig) {
+    return JSON.stringify({
+        baseURL: modelConfig.openaiBaseURL || DEFAULT_OPENAI_BASE_URL,
+        apiKey: modelConfig.openaiApiKey || '',
+        modelName: modelConfig.modelName
+    });
+}
+function buildModelVerifyCurlCommands(configs) {
+    const commandMap = new Map();
+    for (const item of configs){
+        const key = buildCurlDedupKey(item.modelConfig);
+        const existing = commandMap.get(key);
+        if (existing) {
+            existing.intents.push(item.intent);
+            continue;
+        }
+        commandMap.set(key, {
+            intents: [
+                item.intent
+            ],
+            curl: buildCurlCommand(item.modelConfig),
+            usesDefaultBaseURL: !item.modelConfig.openaiBaseURL
+        });
+    }
+    return [
+        ...commandMap.values()
+    ];
+}
+function formatModelVerifyFailureOutput(message, curlCommands) {
+    const details = message?.trim() || 'No failure details were generated.';
+    const curlSection = curlCommands.map((item)=>{
+        const baseUrlNote = item.usesDefaultBaseURL ? ' (base URL not configured; using OpenAI SDK default)' : '';
+        return chalk.gray(`# ${item.intents.join(', ')}${baseUrlNote}\n${item.curl}`);
+    }).join('\n\n');
+    return [
+        chalk.red.bold('❌ Model verify failed with messages:'),
+        MODEL_VERIFY_SEPARATOR,
+        '',
+        details,
+        '',
+        MODEL_VERIFY_SEPARATOR,
+        'Generated curl requests for basic API connectivity:',
+        'If the error is a basic connectivity issue, use these requests to test the base URL, API key, and model name directly.',
+        'These commands contain your API key. Do not share them publicly.',
+        '',
+        curlSection
+    ].join('\n');
+}
+function getDefaultDeps(io) {
+    return {
+        loadDotenv: ()=>{
+            loadDotenvConfig({
+                dotenvDebug: true,
+                dotenvOverride: true,
+                log: io.stdout
+            });
+        },
+        getModelConfig: (intent)=>globalModelConfigManager.getModelConfig(intent),
+        verifyModel: runConnectivityTest
+    };
+}
+async function runModelVerifyCommand(args, deps, io, commandName) {
+    try {
+        if (args.includes('--help') || args.includes('-h')) {
+            io.stdout(modelCommandUsage(commandName));
+            return 0;
+        }
+        assertNoModelVerifyOptions(args, commandName);
+        io.stdout('Model verify started. This usually completes within 30 seconds.');
+        deps.loadDotenv();
+        io.stdout('');
+        const defaultModelConfig = deps.getModelConfig('default');
+        const planningModelConfig = deps.getModelConfig('planning');
+        const insightModelConfig = deps.getModelConfig('insight');
+        const curlCommands = buildModelVerifyCurlCommands([
+            {
+                intent: 'default',
+                modelConfig: defaultModelConfig
+            },
+            {
+                intent: 'planning',
+                modelConfig: planningModelConfig
+            },
+            {
+                intent: 'insight',
+                modelConfig: insightModelConfig
+            }
+        ]);
+        const baseURLs = new Map();
+        const modelRows = [
+            [
+                'planning',
+                planningModelConfig,
+                'Text response'
+            ],
+            [
+                'insight',
+                insightModelConfig,
+                'Image understanding'
+            ],
+            [
+                'default',
+                defaultModelConfig,
+                'AI locate'
+            ]
+        ].map(([intent, config, check])=>{
+            const baseURL = config.openaiExtraConfig?.baseURL ?? config.openaiBaseURL;
+            const endpoint = baseURL ? displayEndpoint(String(baseURL)) : 'provider default';
+            if (!baseURLs.has(endpoint)) baseURLs.set(endpoint, baseURLs.size + 1);
+            return [
+                intent,
+                config.modelName,
+                check,
+                `[${baseURLs.get(endpoint)}]`
+            ];
+        });
+        io.stdout(formatTable([
+            'Model role',
+            'Model',
+            'Check',
+            'Endpoint'
+        ], modelRows));
+        io.stdout('');
+        for (const [endpoint, index] of baseURLs)io.stdout(`[${index}] Base URL: ${endpoint}`);
+        io.stdout('\nRunning 3 checks...\n');
+        const result = await deps.verifyModel({
+            defaultModelConfig,
+            planningModelConfig,
+            insightModelConfig
+        });
+        const checkLabels = {
+            text: 'Text response',
+            vision: 'Image understanding',
+            aiLocate: 'AI locate'
+        };
+        io.stdout(formatTable([
+            'Model role',
+            'Check',
+            'Result',
+            'Requests',
+            'Time'
+        ], result.checks.map((check)=>[
+                check.intent,
+                checkLabels[check.name],
+                check.passed ? 'PASS' : 'FAIL',
+                String(check.requestCount),
+                `${check.durationMs} ms`
+            ])));
+        const requestEndpoints = new Map();
+        for (const check of result.checks){
+            for (const endpoint of check.endpoints){
+                const display = displayEndpoint(endpoint);
+                if (!requestEndpoints.has(display)) requestEndpoints.set(display, new Set());
+                requestEndpoints.get(display).add(check.intent);
+            }
+            if (!check.passed && check.message) io.stdout(`\n${check.intent}: ${check.message}`);
+        }
+        if (requestEndpoints.size > 0) {
+            io.stdout('\nRequest endpoints:');
+            for (const [endpoint, intents] of requestEndpoints)io.stdout(`  ${[
+                ...intents
+            ].join(', ')}: ${endpoint}`);
+        }
+        io.stdout('');
+        const passedChecks = result.checks.filter((check)=>check.passed).length;
+        io.stdout(`Requests sent: ${result.requestCount}. Checks: ${passedChecks} passed, ${result.checks.length - passedChecks} failed.`);
+        if (result.passed) {
+            io.stdout('✅ Model verify passed.');
+            return 0;
+        }
+        io.stderr(formatModelVerifyFailureOutput(result.message, curlCommands));
+        return 1;
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        io.stderr([
+            MODEL_VERIFY_SEPARATOR,
+            '❌ Model verify failed with messages:',
+            '',
+            message,
+            MODEL_VERIFY_SEPARATOR
+        ].join('\n'));
+        return 1;
+    }
+}
+async function runModelCommand(rawArgs, deps, io = {
+    stdout: console.log,
+    stderr: console.error
+}, commandName = 'midscene-test') {
+    const [, action, ...restArgs] = rawArgs;
+    const mergedDeps = {
+        ...getDefaultDeps(io),
+        ...deps
+    };
+    if (!action || '--help' === action || '-h' === action) {
+        io.stdout(modelCommandUsage(commandName));
+        return 0;
+    }
+    if ('verify' === action) return runModelVerifyCommand(restArgs, mergedDeps, io, commandName);
+    if ('eval' === action) {
+        io.stderr(`${commandName} model eval is not implemented yet. It is reserved for future model evaluation suites.`);
+        return 1;
+    }
+    io.stderr(`Unknown ${commandName} model command: ${action}\n\n${modelCommandUsage(commandName)}`);
+    return 1;
+}
+export { buildModelVerifyCurlCommands, loadDotenvConfig, runModelCommand };
+
+//# sourceMappingURL=model-command.mjs.map

package/dist/es/cli/model-command.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli/model-command.mjs","sources":["../../../src/cli/dotenv-loader.ts","../../../src/cli/model-command.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport dotenv from 'dotenv';\n\nexport interface DotenvLoadOptions {\n  dotenvOverride?: boolean;\n  dotenvDebug?: boolean;\n  cwd?: string;\n  log?: (message: string) => void;\n}\n\nexport function loadDotenvConfig(options: DotenvLoadOptions = {}) {\n  const dotEnvConfigFile = join(options.cwd ?? process.cwd(), '.env');\n  if (!existsSync(dotEnvConfigFile)) {\n    return;\n  }\n\n  options.log?.(`   Env file: ${dotEnvConfigFile}`);\n  dotenv.config({\n    path: dotEnvConfigFile,\n    debug: options.dotenvDebug,\n    override: options.dotenvOverride,\n  });\n}\n","import { runConnectivityTest } from '@midscene/core';\nimport {\n  type IModelConfig,\n  type TIntent,\n  globalModelConfigManager,\n} from '@midscene/shared/env';\nimport chalk from 'chalk';\nimport { loadDotenvConfig } from './dotenv-loader';\n\nconst DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1';\nconst MODEL_VERIFY_SEPARATOR = '────────────────────────────────────────';\nconst modelCommandUsage = (commandName: string) => `Usage:\n  ${commandName} model verify\n\nVerify model connectivity and Midscene compatibility for default, planning, and insight models.\nReads .env from the current working directory, overriding shell environment variables.\nShows endpoints, model names, request counts, and per-check results with duration.\nReturns exit code 0 on success and 1 on failure.\n\nOptions:\n  --help, -h  Show this help\n`;\n\ninterface ModelCommandIO {\n  stdout: (message: string) => void;\n  stderr: (message: string) => void;\n}\n\ninterface ModelCommandDeps {\n  loadDotenv: () => void;\n  getModelConfig: (intent: TIntent) => IModelConfig;\n  verifyModel: typeof runConnectivityTest;\n}\n\nexport interface CurlCommandItem {\n  intents: TIntent[];\n  curl: string;\n  usesDefaultBaseURL: boolean;\n}\n\nfunction assertNoModelVerifyOptions(args: string[], commandName: string) {\n  for (const arg of args) {\n    throw new Error(`Unknown option for ${commandName} model verify: ${arg}`);\n  }\n}\n\nfunction formatTable(headers: string[], rows: string[][]): string {\n  const widths = headers.map((header, index) =>\n    Math.max(header.length, ...rows.map((row) => row[index].length)),\n  );\n  const formatRow = (row: string[]) =>\n    row\n      .map((cell, index) => cell.padEnd(widths[index]))\n      .join('  ')\n      .trimEnd();\n  return [formatRow(headers), ...rows.map(formatRow)].join('\\n');\n}\n\nfunction displayEndpoint(endpoint: string): string {\n  try {\n    const url = new URL(endpoint);\n    url.username = '';\n    url.password = '';\n    url.search = '';\n    url.hash = '';\n    return url.toString();\n  } catch {\n    return '(invalid endpoint URL)';\n  }\n}\n\nfunction shellSingleQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`;\n}\n\nfunction buildChatCompletionsUrl(baseURL?: string): string {\n  const normalizedBaseURL = (baseURL || DEFAULT_OPENAI_BASE_URL).replace(\n    /\\/+$/,\n    '',\n  );\n  if (normalizedBaseURL.endsWith('/chat/completions')) {\n    return normalizedBaseURL;\n  }\n  return `${normalizedBaseURL}/chat/completions`;\n}\n\nfunction buildCurlCommand(modelConfig: IModelConfig): string {\n  const payload = {\n    model: modelConfig.modelName,\n    messages: [{ role: 'user', content: 'What is 1+1?' }],\n  };\n\n  return [\n    `curl -X POST ${shellSingleQuote(buildChatCompletionsUrl(modelConfig.openaiBaseURL))} \\\\`,\n    `  -H ${shellSingleQuote(`Authorization: Bearer ${modelConfig.openaiApiKey || ''}`)} \\\\`,\n    `  -H ${shellSingleQuote('Content-Type: application/json')} \\\\`,\n    `  -d ${shellSingleQuote(JSON.stringify(payload, null, 2))}`,\n  ].join('\\n');\n}\n\nfunction buildCurlDedupKey(modelConfig: IModelConfig): string {\n  return JSON.stringify({\n    baseURL: modelConfig.openaiBaseURL || DEFAULT_OPENAI_BASE_URL,\n    apiKey: modelConfig.openaiApiKey || '',\n    modelName: modelConfig.modelName,\n  });\n}\n\nexport function buildModelVerifyCurlCommands(\n  configs: Array<{ intent: TIntent; modelConfig: IModelConfig }>,\n): CurlCommandItem[] {\n  const commandMap = new Map<string, CurlCommandItem>();\n\n  for (const item of configs) {\n    const key = buildCurlDedupKey(item.modelConfig);\n    const existing = commandMap.get(key);\n    if (existing) {\n      existing.intents.push(item.intent);\n      continue;\n    }\n    commandMap.set(key, {\n      intents: [item.intent],\n      curl: buildCurlCommand(item.modelConfig),\n      usesDefaultBaseURL: !item.modelConfig.openaiBaseURL,\n    });\n  }\n\n  return [...commandMap.values()];\n}\n\nfunction formatModelVerifyFailureOutput(\n  message: string | undefined,\n  curlCommands: CurlCommandItem[],\n): string {\n  const details = message?.trim() || 'No failure details were generated.';\n  const curlSection = curlCommands\n    .map((item) => {\n      const baseUrlNote = item.usesDefaultBaseURL\n        ? ' (base URL not configured; using OpenAI SDK default)'\n        : '';\n      return chalk.gray(\n        `# ${item.intents.join(', ')}${baseUrlNote}\\n${item.curl}`,\n      );\n    })\n    .join('\\n\\n');\n\n  return [\n    chalk.red.bold('❌ Model verify failed with messages:'),\n    MODEL_VERIFY_SEPARATOR,\n    '',\n    details,\n    '',\n    MODEL_VERIFY_SEPARATOR,\n    'Generated curl requests for basic API connectivity:',\n    'If the error is a basic connectivity issue, use these requests to test the base URL, API key, and model name directly.',\n    'These commands contain your API key. Do not share them publicly.',\n    '',\n    curlSection,\n  ].join('\\n');\n}\n\nfunction getDefaultDeps(io: ModelCommandIO): ModelCommandDeps {\n  return {\n    loadDotenv: () => {\n      loadDotenvConfig({\n        dotenvDebug: true,\n        dotenvOverride: true,\n        log: io.stdout,\n      });\n    },\n    getModelConfig: (intent) => globalModelConfigManager.getModelConfig(intent),\n    verifyModel: runConnectivityTest,\n  };\n}\n\nasync function runModelVerifyCommand(\n  args: string[],\n  deps: ModelCommandDeps,\n  io: ModelCommandIO,\n  commandName: string,\n): Promise<number> {\n  try {\n    if (args.includes('--help') || args.includes('-h')) {\n      io.stdout(modelCommandUsage(commandName));\n      return 0;\n    }\n\n    assertNoModelVerifyOptions(args, commandName);\n    io.stdout(\n      'Model verify started. This usually completes within 30 seconds.',\n    );\n    deps.loadDotenv();\n    io.stdout('');\n\n    const defaultModelConfig = deps.getModelConfig('default');\n    const planningModelConfig = deps.getModelConfig('planning');\n    const insightModelConfig = deps.getModelConfig('insight');\n    const curlCommands = buildModelVerifyCurlCommands([\n      { intent: 'default', modelConfig: defaultModelConfig },\n      { intent: 'planning', modelConfig: planningModelConfig },\n      { intent: 'insight', modelConfig: insightModelConfig },\n    ]);\n\n    const baseURLs = new Map<string, number>();\n    const modelRows = (\n      [\n        ['planning', planningModelConfig, 'Text response'],\n        ['insight', insightModelConfig, 'Image understanding'],\n        ['default', defaultModelConfig, 'AI locate'],\n      ] as const\n    ).map(([intent, config, check]) => {\n      const baseURL = config.openaiExtraConfig?.baseURL ?? config.openaiBaseURL;\n      const endpoint = baseURL\n        ? displayEndpoint(String(baseURL))\n        : 'provider default';\n      if (!baseURLs.has(endpoint)) baseURLs.set(endpoint, baseURLs.size + 1);\n      return [intent, config.modelName, check, `[${baseURLs.get(endpoint)}]`];\n    });\n    io.stdout(\n      formatTable(['Model role', 'Model', 'Check', 'Endpoint'], modelRows),\n    );\n    io.stdout('');\n    for (const [endpoint, index] of baseURLs) {\n      io.stdout(`[${index}] Base URL: ${endpoint}`);\n    }\n    io.stdout('\\nRunning 3 checks...\\n');\n\n    const result = await deps.verifyModel({\n      defaultModelConfig,\n      planningModelConfig,\n      insightModelConfig,\n    });\n\n    const checkLabels = {\n      text: 'Text response',\n      vision: 'Image understanding',\n      aiLocate: 'AI locate',\n    };\n    io.stdout(\n      formatTable(\n        ['Model role', 'Check', 'Result', 'Requests', 'Time'],\n        result.checks.map((check) => [\n          check.intent,\n          checkLabels[check.name],\n          check.passed ? 'PASS' : 'FAIL',\n          String(check.requestCount),\n          `${check.durationMs} ms`,\n        ]),\n      ),\n    );\n    const requestEndpoints = new Map<string, Set<string>>();\n    for (const check of result.checks) {\n      for (const endpoint of check.endpoints) {\n        const display = displayEndpoint(endpoint);\n        if (!requestEndpoints.has(display))\n          requestEndpoints.set(display, new Set());\n        requestEndpoints.get(display)!.add(check.intent);\n      }\n      if (!check.passed && check.message)\n        io.stdout(`\\n${check.intent}: ${check.message}`);\n    }\n    if (requestEndpoints.size > 0) {\n      io.stdout('\\nRequest endpoints:');\n      for (const [endpoint, intents] of requestEndpoints) {\n        io.stdout(`  ${[...intents].join(', ')}: ${endpoint}`);\n      }\n    }\n    io.stdout('');\n    const passedChecks = result.checks.filter((check) => check.passed).length;\n    io.stdout(\n      `Requests sent: ${result.requestCount}. Checks: ${passedChecks} passed, ${result.checks.length - passedChecks} failed.`,\n    );\n\n    if (result.passed) {\n      io.stdout('✅ Model verify passed.');\n      return 0;\n    }\n\n    io.stderr(formatModelVerifyFailureOutput(result.message, curlCommands));\n    return 1;\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    io.stderr(\n      [\n        MODEL_VERIFY_SEPARATOR,\n        '❌ Model verify failed with messages:',\n        '',\n        message,\n        MODEL_VERIFY_SEPARATOR,\n      ].join('\\n'),\n    );\n    return 1;\n  }\n}\n\nexport async function runModelCommand(\n  rawArgs: string[],\n  deps?: Partial<ModelCommandDeps>,\n  io: ModelCommandIO = {\n    stdout: console.log,\n    stderr: console.error,\n  },\n  commandName = 'midscene-test',\n): Promise<number> {\n  const [, action, ...restArgs] = rawArgs;\n  const mergedDeps = {\n    ...getDefaultDeps(io),\n    ...deps,\n  };\n\n  if (!action || action === '--help' || action === '-h') {\n    io.stdout(modelCommandUsage(commandName));\n    return 0;\n  }\n\n  if (action === 'verify') {\n    return runModelVerifyCommand(restArgs, mergedDeps, io, commandName);\n  }\n\n  if (action === 'eval') {\n    io.stderr(\n      `${commandName} model eval is not implemented yet. It is reserved for future model evaluation suites.`,\n    );\n    return 1;\n  }\n\n  io.stderr(\n    `Unknown ${commandName} model command: ${action}\\n\\n${modelCommandUsage(commandName)}`,\n  );\n  return 1;\n}\n\nexport { loadDotenvConfig, type DotenvLoadOptions } from './dotenv-loader';\n"],"names":["loadDotenvConfig","options","dotEnvConfigFile","join","process","existsSync","dotenv","DEFAULT_OPENAI_BASE_URL","MODEL_VERIFY_SEPARATOR","modelCommandUsage","commandName","assertNoModelVerifyOptions","args","arg","Error","formatTable","headers","rows","widths","header","index","Math","row","formatRow","cell","displayEndpoint","endpoint","url","URL","shellSingleQuote","value","buildChatCompletionsUrl","baseURL","normalizedBaseURL","buildCurlCommand","modelConfig","payload","JSON","buildCurlDedupKey","buildModelVerifyCurlCommands","configs","commandMap","Map","item","key","existing","formatModelVerifyFailureOutput","message","curlCommands","details","curlSection","baseUrlNote","chalk","getDefaultDeps","io","intent","globalModelConfigManager","runConnectivityTest","runModelVerifyCommand","deps","defaultModelConfig","planningModelConfig","insightModelConfig","baseURLs","modelRows","config","check","String","result","checkLabels","requestEndpoints","display","Set","intents","passedChecks","error","runModelCommand","rawArgs","console","action","restArgs","mergedDeps"],"mappings":";;;;;;AAWO,SAASA,iBAAiBC,UAA6B,CAAC,CAAC;IAC9D,MAAMC,mBAAmBC,KAAKF,QAAQ,GAAG,IAAIG,QAAQ,GAAG,IAAI;IAC5D,IAAI,CAACC,WAAWH,mBACd;IAGFD,QAAQ,GAAG,GAAG,CAAC,aAAa,EAAEC,kBAAkB;IAChDI,OAAO,MAAM,CAAC;QACZ,MAAMJ;QACN,OAAOD,QAAQ,WAAW;QAC1B,UAAUA,QAAQ,cAAc;IAClC;AACF;ACdA,MAAMM,0BAA0B;AAChC,MAAMC,yBAAyB;AAC/B,MAAMC,oBAAoB,CAACC,cAAwB,CAAC;EAClD,EAAEA,YAAY;;;;;;;;;AAShB,CAAC;AAmBD,SAASC,2BAA2BC,IAAc,EAAEF,WAAmB;IACrE,KAAK,MAAMG,OAAOD,KAChB,MAAM,IAAIE,MAAM,CAAC,mBAAmB,EAAEJ,YAAY,eAAe,EAAEG,KAAK;AAE5E;AAEA,SAASE,YAAYC,OAAiB,EAAEC,IAAgB;IACtD,MAAMC,SAASF,QAAQ,GAAG,CAAC,CAACG,QAAQC,QAClCC,KAAK,GAAG,CAACF,OAAO,MAAM,KAAKF,KAAK,GAAG,CAAC,CAACK,MAAQA,GAAG,CAACF,MAAM,CAAC,MAAM;IAEhE,MAAMG,YAAY,CAACD,MACjBA,IACG,GAAG,CAAC,CAACE,MAAMJ,QAAUI,KAAK,MAAM,CAACN,MAAM,CAACE,MAAM,GAC9C,IAAI,CAAC,MACL,OAAO;IACZ,OAAO;QAACG,UAAUP;WAAaC,KAAK,GAAG,CAACM;KAAW,CAAC,IAAI,CAAC;AAC3D;AAEA,SAASE,gBAAgBC,QAAgB;IACvC,IAAI;QACF,MAAMC,MAAM,IAAIC,IAAIF;QACpBC,IAAI,QAAQ,GAAG;QACfA,IAAI,QAAQ,GAAG;QACfA,IAAI,MAAM,GAAG;QACbA,IAAI,IAAI,GAAG;QACX,OAAOA,IAAI,QAAQ;IACrB,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE,iBAAiBC,KAAa;IACrC,OAAO,CAAC,CAAC,EAAEA,MAAM,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;AAC5C;AAEA,SAASC,wBAAwBC,OAAgB;IAC/C,MAAMC,oBAAqBD,AAAAA,CAAAA,WAAWzB,uBAAsB,EAAG,OAAO,CACpE,QACA;IAEF,IAAI0B,kBAAkB,QAAQ,CAAC,sBAC7B,OAAOA;IAET,OAAO,GAAGA,kBAAkB,iBAAiB,CAAC;AAChD;AAEA,SAASC,iBAAiBC,WAAyB;IACjD,MAAMC,UAAU;QACd,OAAOD,YAAY,SAAS;QAC5B,UAAU;YAAC;gBAAE,MAAM;gBAAQ,SAAS;YAAe;SAAE;IACvD;IAEA,OAAO;QACL,CAAC,aAAa,EAAEN,iBAAiBE,wBAAwBI,YAAY,aAAa,GAAG,GAAG,CAAC;QACzF,CAAC,KAAK,EAAEN,iBAAiB,CAAC,sBAAsB,EAAEM,YAAY,YAAY,IAAI,IAAI,EAAE,GAAG,CAAC;QACxF,CAAC,KAAK,EAAEN,iBAAiB,kCAAkC,GAAG,CAAC;QAC/D,CAAC,KAAK,EAAEA,iBAAiBQ,KAAK,SAAS,CAACD,SAAS,MAAM,KAAK;KAC7D,CAAC,IAAI,CAAC;AACT;AAEA,SAASE,kBAAkBH,WAAyB;IAClD,OAAOE,KAAK,SAAS,CAAC;QACpB,SAASF,YAAY,aAAa,IAAI5B;QACtC,QAAQ4B,YAAY,YAAY,IAAI;QACpC,WAAWA,YAAY,SAAS;IAClC;AACF;AAEO,SAASI,6BACdC,OAA8D;IAE9D,MAAMC,aAAa,IAAIC;IAEvB,KAAK,MAAMC,QAAQH,QAAS;QAC1B,MAAMI,MAAMN,kBAAkBK,KAAK,WAAW;QAC9C,MAAME,WAAWJ,WAAW,GAAG,CAACG;QAChC,IAAIC,UAAU;YACZA,SAAS,OAAO,CAAC,IAAI,CAACF,KAAK,MAAM;YACjC;QACF;QACAF,WAAW,GAAG,CAACG,KAAK;YAClB,SAAS;gBAACD,KAAK,MAAM;aAAC;YACtB,MAAMT,iBAAiBS,KAAK,WAAW;YACvC,oBAAoB,CAACA,KAAK,WAAW,CAAC,aAAa;QACrD;IACF;IAEA,OAAO;WAAIF,WAAW,MAAM;KAAG;AACjC;AAEA,SAASK,+BACPC,OAA2B,EAC3BC,YAA+B;IAE/B,MAAMC,UAAUF,SAAS,UAAU;IACnC,MAAMG,cAAcF,aACjB,GAAG,CAAC,CAACL;QACJ,MAAMQ,cAAcR,KAAK,kBAAkB,GACvC,yDACA;QACJ,OAAOS,MAAM,IAAI,CACf,CAAC,EAAE,EAAET,KAAK,OAAO,CAAC,IAAI,CAAC,QAAQQ,YAAY,EAAE,EAAER,KAAK,IAAI,EAAE;IAE9D,GACC,IAAI,CAAC;IAER,OAAO;QACLS,MAAM,GAAG,CAAC,IAAI,CAAC;QACf5C;QACA;QACAyC;QACA;QACAzC;QACA;QACA;QACA;QACA;QACA0C;KACD,CAAC,IAAI,CAAC;AACT;AAEA,SAASG,eAAeC,EAAkB;IACxC,OAAO;QACL,YAAY;YACVtD,iBAAiB;gBACf,aAAa;gBACb,gBAAgB;gBAChB,KAAKsD,GAAG,MAAM;YAChB;QACF;QACA,gBAAgB,CAACC,SAAWC,yBAAyB,cAAc,CAACD;QACpE,aAAaE;IACf;AACF;AAEA,eAAeC,sBACb9C,IAAc,EACd+C,IAAsB,EACtBL,EAAkB,EAClB5C,WAAmB;IAEnB,IAAI;QACF,IAAIE,KAAK,QAAQ,CAAC,aAAaA,KAAK,QAAQ,CAAC,OAAO;YAClD0C,GAAG,MAAM,CAAC7C,kBAAkBC;YAC5B,OAAO;QACT;QAEAC,2BAA2BC,MAAMF;QACjC4C,GAAG,MAAM,CACP;QAEFK,KAAK,UAAU;QACfL,GAAG,MAAM,CAAC;QAEV,MAAMM,qBAAqBD,KAAK,cAAc,CAAC;QAC/C,MAAME,sBAAsBF,KAAK,cAAc,CAAC;QAChD,MAAMG,qBAAqBH,KAAK,cAAc,CAAC;QAC/C,MAAMX,eAAeT,6BAA6B;YAChD;gBAAE,QAAQ;gBAAW,aAAaqB;YAAmB;YACrD;gBAAE,QAAQ;gBAAY,aAAaC;YAAoB;YACvD;gBAAE,QAAQ;gBAAW,aAAaC;YAAmB;SACtD;QAED,MAAMC,WAAW,IAAIrB;QACrB,MAAMsB,YACJ;YACE;gBAAC;gBAAYH;gBAAqB;aAAgB;YAClD;gBAAC;gBAAWC;gBAAoB;aAAsB;YACtD;gBAAC;gBAAWF;gBAAoB;aAAY;SAC7C,CACD,GAAG,CAAC,CAAC,CAACL,QAAQU,QAAQC,MAAM;YAC5B,MAAMlC,UAAUiC,OAAO,iBAAiB,EAAE,WAAWA,OAAO,aAAa;YACzE,MAAMvC,WAAWM,UACbP,gBAAgB0C,OAAOnC,YACvB;YACJ,IAAI,CAAC+B,SAAS,GAAG,CAACrC,WAAWqC,SAAS,GAAG,CAACrC,UAAUqC,SAAS,IAAI,GAAG;YACpE,OAAO;gBAACR;gBAAQU,OAAO,SAAS;gBAAEC;gBAAO,CAAC,CAAC,EAAEH,SAAS,GAAG,CAACrC,UAAU,CAAC,CAAC;aAAC;QACzE;QACA4B,GAAG,MAAM,CACPvC,YAAY;YAAC;YAAc;YAAS;YAAS;SAAW,EAAEiD;QAE5DV,GAAG,MAAM,CAAC;QACV,KAAK,MAAM,CAAC5B,UAAUN,MAAM,IAAI2C,SAC9BT,GAAG,MAAM,CAAC,CAAC,CAAC,EAAElC,MAAM,YAAY,EAAEM,UAAU;QAE9C4B,GAAG,MAAM,CAAC;QAEV,MAAMc,SAAS,MAAMT,KAAK,WAAW,CAAC;YACpCC;YACAC;YACAC;QACF;QAEA,MAAMO,cAAc;YAClB,MAAM;YACN,QAAQ;YACR,UAAU;QACZ;QACAf,GAAG,MAAM,CACPvC,YACE;YAAC;YAAc;YAAS;YAAU;YAAY;SAAO,EACrDqD,OAAO,MAAM,CAAC,GAAG,CAAC,CAACF,QAAU;gBAC3BA,MAAM,MAAM;gBACZG,WAAW,CAACH,MAAM,IAAI,CAAC;gBACvBA,MAAM,MAAM,GAAG,SAAS;gBACxBC,OAAOD,MAAM,YAAY;gBACzB,GAAGA,MAAM,UAAU,CAAC,GAAG,CAAC;aACzB;QAGL,MAAMI,mBAAmB,IAAI5B;QAC7B,KAAK,MAAMwB,SAASE,OAAO,MAAM,CAAE;YACjC,KAAK,MAAM1C,YAAYwC,MAAM,SAAS,CAAE;gBACtC,MAAMK,UAAU9C,gBAAgBC;gBAChC,IAAI,CAAC4C,iBAAiB,GAAG,CAACC,UACxBD,iBAAiB,GAAG,CAACC,SAAS,IAAIC;gBACpCF,iBAAiB,GAAG,CAACC,SAAU,GAAG,CAACL,MAAM,MAAM;YACjD;YACA,IAAI,CAACA,MAAM,MAAM,IAAIA,MAAM,OAAO,EAChCZ,GAAG,MAAM,CAAC,CAAC,EAAE,EAAEY,MAAM,MAAM,CAAC,EAAE,EAAEA,MAAM,OAAO,EAAE;QACnD;QACA,IAAII,iBAAiB,IAAI,GAAG,GAAG;YAC7BhB,GAAG,MAAM,CAAC;YACV,KAAK,MAAM,CAAC5B,UAAU+C,QAAQ,IAAIH,iBAChChB,GAAG,MAAM,CAAC,CAAC,EAAE,EAAE;mBAAImB;aAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE/C,UAAU;QAEzD;QACA4B,GAAG,MAAM,CAAC;QACV,MAAMoB,eAAeN,OAAO,MAAM,CAAC,MAAM,CAAC,CAACF,QAAUA,MAAM,MAAM,EAAE,MAAM;QACzEZ,GAAG,MAAM,CACP,CAAC,eAAe,EAAEc,OAAO,YAAY,CAAC,UAAU,EAAEM,aAAa,SAAS,EAAEN,OAAO,MAAM,CAAC,MAAM,GAAGM,aAAa,QAAQ,CAAC;QAGzH,IAAIN,OAAO,MAAM,EAAE;YACjBd,GAAG,MAAM,CAAC;YACV,OAAO;QACT;QAEAA,GAAG,MAAM,CAACR,+BAA+BsB,OAAO,OAAO,EAAEpB;QACzD,OAAO;IACT,EAAE,OAAO2B,OAAO;QACd,MAAM5B,UAAU4B,iBAAiB7D,QAAQ6D,MAAM,OAAO,GAAGR,OAAOQ;QAChErB,GAAG,MAAM,CACP;YACE9C;YACA;YACA;YACAuC;YACAvC;SACD,CAAC,IAAI,CAAC;QAET,OAAO;IACT;AACF;AAEO,eAAeoE,gBACpBC,OAAiB,EACjBlB,IAAgC,EAChCL,KAAqB;IACnB,QAAQwB,QAAQ,GAAG;IACnB,QAAQA,QAAQ,KAAK;AACvB,CAAC,EACDpE,cAAc,eAAe;IAE7B,MAAM,GAAGqE,QAAQ,GAAGC,SAAS,GAAGH;IAChC,MAAMI,aAAa;QACjB,GAAG5B,eAAeC,GAAG;QACrB,GAAGK,IAAI;IACT;IAEA,IAAI,CAACoB,UAAUA,AAAW,aAAXA,UAAuBA,AAAW,SAAXA,QAAiB;QACrDzB,GAAG,MAAM,CAAC7C,kBAAkBC;QAC5B,OAAO;IACT;IAEA,IAAIqE,AAAW,aAAXA,QACF,OAAOrB,sBAAsBsB,UAAUC,YAAY3B,IAAI5C;IAGzD,IAAIqE,AAAW,WAAXA,QAAmB;QACrBzB,GAAG,MAAM,CACP,GAAG5C,YAAY,sFAAsF,CAAC;QAExG,OAAO;IACT;IAEA4C,GAAG,MAAM,CACP,CAAC,QAAQ,EAAE5C,YAAY,gBAAgB,EAAEqE,OAAO,IAAI,EAAEtE,kBAAkBC,cAAc;IAExF,OAAO;AACT"}

package/dist/es/test-cli.mjs

@@ -1,10 +1,11 @@
 import * as __rspack_external_node_fs_5ea92f0c from "node:fs";
 import * as __rspack_external_node_path_c5b9b54f from "node:path";
+import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, isFatalDeviceError, normalizeSteps, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument } from "@midscene/core/internal/test-runner";
+import { inspect } from "node:util";
 import { z } from "zod/v4";
 import { pathToFileURL } from "node:url";
 import { require as api_require } from "tsx/cjs/api";
 import { tsImport } from "tsx/esm/api";
-import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, isFatalDeviceError, normalizeSteps, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument } from "@midscene/core/internal/test-runner";
 import { globSync } from "tinyglobby";
 import { JSON_SCHEMA, load } from "js-yaml";
 import { createHash, randomUUID } from "node:crypto";
@@ -21,7 +22,7 @@ var __webpack_modules__ = {
     },
     "./package.json" (module) {
         module.exports = {
-            rE: "1.14.0"
+            rE: "1.15.0"
         };
     }
 };
@@ -97,6 +98,11 @@ __webpack_require__.m = __webpack_modules__;
 var external_node_fs_ = __webpack_require__("node:fs");
 var external_node_path_ = __webpack_require__("node:path");
 var package_0 = __webpack_require__("./package.json");
+const formatCliError = (error)=>'string' == typeof error ? error : inspect(error, {
+        colors: false,
+        depth: null,
+        customInspect: false
+    });
 const GENERATED_COMMENT = '<!-- Generated by `midscene-test nodes`. Do not edit directly. -->';
 const sortNodesForSpec = (nodes)=>{
     const priority = (name)=>'aiAct' === name ? 0 : 'aiAssert' === name ? 1 : 2;
@@ -1580,6 +1586,7 @@ Usage:
   midscene-test --config <midscene.config.ts> [options]
   midscene-test nodes [directory] [--config midscene.config.ts]
   midscene-test create --help
+  midscene-test model verify    Check model connectivity and Midscene compatibility
 
 Options:
   --config <path>              Native TypeScript or JavaScript Test config
@@ -1645,8 +1652,78 @@ const runNodesCommand = async (options, io)=>{
         io.log(`\nNode Spec generated: ${specPath}`);
     }
 };
+const reportTestProjectResult = (result, io, infrastructureErrors = result.errors ?? [])=>{
+    const reportedErrors = new Set();
+    const reportError = (source, error)=>{
+        if (error instanceof Error && reportedErrors.has(error)) return;
+        io.error(`midscene-test: ${source}: ${formatCliError(error)}`);
+        const pending = [
+            error
+        ];
+        while(pending.length > 0){
+            const current = pending.pop();
+            if (!(current instanceof Error) || reportedErrors.has(current)) continue;
+            reportedErrors.add(current);
+            pending.push(current.cause);
+            if (current instanceof AggregateError) pending.push(...current.errors);
+        }
+    };
+    for (const failure of result.collectionErrors)reportError(`${failure.projectName}/${failure.sourcePath}`, failure.error);
+    const finalCases = new Map(result.cases.map((outcome)=>[
+            outcome.caseId,
+            outcome
+        ]));
+    for (const outcome of finalCases.values()){
+        if ('failed' !== outcome.status || !outcome.run) continue;
+        const run = outcome.run;
+        const source = `${run.projectName}/${run.sourcePath} / ${run.name}`;
+        for (const step of [
+            ...run.beforeEach,
+            ...run.steps,
+            ...run.afterEach
+        ])if (step.error) reportError(`${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}`, step.error);
+        for (const error of [
+            ...run.executionErrors ?? [],
+            ...run.teardownErrors ?? []
+        ])reportError(source, error);
+    }
+    const finalDocuments = new Map(result.documents.map((document)=>[
+            document.documentId,
+            document
+        ]));
+    for (const document of finalDocuments.values()){
+        if ('failed' !== document.status) continue;
+        const source = `${document.projectName}/${document.sourcePath}`;
+        for (const step of [
+            ...document.beforeAll,
+            ...document.afterAll
+        ])if (step.error) reportError(`${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}`, step.error);
+        for (const error of [
+            ...document.executionErrors ?? [],
+            ...document.teardownErrors ?? []
+        ])reportError(source, error);
+        for (const failure of document.hostErrors ?? [])reportError(`${source} / host ${failure.phase}`, failure.error);
+    }
+    for (const project of result.projects){
+        if (project.lifecycle?.setupError) reportError(`${project.name} / setup`, project.lifecycle.setupError);
+        for (const error of project.lifecycle?.teardownErrors ?? [])reportError(`${project.name} / teardown`, error);
+    }
+    for (const error of infrastructureErrors)reportError('infrastructure', error);
+    io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
+    io.log(`Results: ${result.resultDir}`);
+    io.log(`Summary: ${result.summaryPath}`);
+    if (result.reportPath) io.log(`Report: ${result.reportPath}`);
+    return result.exitCode;
+};
 async function runTestCli(args, io = defaultCliIO) {
     try {
+        if ('model' === args[0]) {
+            const { runModelCommand } = await __webpack_require__.e("77").then(__webpack_require__.bind(__webpack_require__, "./src/cli/model-command.ts?bb10"));
+            return runModelCommand(args, void 0, {
+                stdout: io.log,
+                stderr: io.error
+            });
+        }
         if ('create' === args[0]) {
             const { runCreateCommand } = await __webpack_require__.e("214").then(__webpack_require__.bind(__webpack_require__, "./src/cli/create-command.ts"));
             await runCreateCommand(args.slice(1), io);
@@ -1665,36 +1742,20 @@ async function runTestCli(args, io = defaultCliIO) {
             await runNodesCommand(options, io);
             return 0;
         }
-        const result = await runTestProject({
-            ...options,
-            onProgress: (message)=>io.log(message)
-        });
-        for (const failure of result.collectionErrors)io.error(`midscene-test: ${failure.projectName}/${failure.sourcePath}: ${failure.error.message}`);
-        const finalCases = new Map(result.cases.map((outcome)=>[
-                outcome.caseId,
-                outcome
-            ]));
-        for (const outcome of finalCases.values()){
-            if ('failed' !== outcome.status || !outcome.run) continue;
-            const run = outcome.run;
-            const source = `${run.projectName}/${run.sourcePath} / ${run.name}`;
-            for (const step of [
-                ...run.beforeEach,
-                ...run.steps,
-                ...run.afterEach
-            ])if (step.error) io.error(`midscene-test: ${source} / ${step.phase}[${step.stepIndex + 1}] ${step.node}: ${step.error.message}`);
-            for (const error of [
-                ...run.executionErrors ?? [],
-                ...run.teardownErrors ?? []
-            ])io.error(`midscene-test: ${source}: ${error.message}`);
+        let result;
+        try {
+            result = await runTestProject({
+                ...options,
+                onProgress: (message)=>io.log(message)
+            });
+        } catch (error) {
+            if (!(error instanceof WorkflowExecutionFailure)) throw error;
+            reportTestProjectResult(error.result, io, error.errors);
+            return 1;
         }
-        io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
-        io.log(`Results: ${result.resultDir}`);
-        io.log(`Summary: ${result.summaryPath}`);
-        if (result.reportPath) io.log(`Report: ${result.reportPath}`);
-        return result.exitCode;
+        return reportTestProjectResult(result, io);
     } catch (error) {
-        io.error(error instanceof Error ? error.message : String(error));
+        io.error(formatCliError(error));
         return 1;
     }
 }