@midscene/test 1.16.1 → 1.16.2

package/dist/es/cli/index.mjs

@@ -1,7 +1,7 @@
 import { dirname, isAbsolute, join, relative, resolve as external_node_path_resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import { require as api_require } from "tsx/cjs/api";
-import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, getPendingScopeOperations, getResourceCleanupCompletion, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText, trackResourceOperation, waitForScopeOperations } from "@midscene/core/internal/test-runner";
+import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, getPendingScopeOperations, getResourceCleanupCompletion, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText, sumTestRunReportMetrics as test_runner_sumTestRunReportMetrics, trackResourceOperation, waitForScopeOperations } from "@midscene/core/internal/test-runner";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { globSync } from "tinyglobby";
 import { createHash, randomUUID } from "node:crypto";
@@ -9,7 +9,9 @@ import { JSON_SCHEMA, load } from "js-yaml";
 import { mkdir, writeFile } from "node:fs/promises";
 import { setMaxListeners } from "node:events";
 import { TestRunReportAssembler, calculateTestRunHealth } from "@midscene/core/report";
-import { getDebug } from "@midscene/shared/logger";
+import { getDebug, runWithConsoleLogScope } from "@midscene/shared/logger";
+import "node:util";
+import { globalConfigManager } from "@midscene/shared/env";
 const registry_NodeRegistry = NodeRegistry;
 const projectIdFromIndex = (index)=>`project-${index}`;
 const defineTestProject = (definition)=>definition;
@@ -360,11 +362,13 @@ function parseWorkflowSource(source) {
                 start: state.position + sourceOffset,
                 end: 0,
                 value: null,
-                children: []
+                children: [],
+                kind: null
             });
             const node = stack.pop();
             node.end = state.position + sourceOffset;
             node.value = state.result;
+            node.kind = state.kind;
             stack.at(-1)?.children.push(node);
             nodes.push(node);
             if ('mapping' !== state.kind) return;
@@ -394,6 +398,36 @@ function parseWorkflowSource(source) {
     const value = sanitizeReportText(redacted, 65536);
     return {
         parsed,
+        locate (path) {
+            let node = nodes.at(-1);
+            for (const part of path){
+                if (!node) return;
+                while(1 === node.children.length && node.children[0].value === node.value)node = node.children[0];
+                if ('number' == typeof part && 'sequence' === node.kind) node = node.children[part];
+                else {
+                    if ('string' != typeof part || 'mapping' !== node.kind) return;
+                    const children = node.children;
+                    node = void 0;
+                    for(let index = 0; index < children.length; index += 2)if (children[index].value === part) {
+                        node = children[index + 1];
+                        break;
+                    }
+                }
+            }
+            if (!node) return;
+            let start = node.start;
+            while(start < node.end)if (/\s/.test(source[start])) start++;
+            else if ('#' === source[start]) {
+                const end = source.indexOf('\n', start);
+                if (end < 0) return;
+                start = end + 1;
+            } else break;
+            const prefix = source.slice(0, start).split(/\r\n|\r|\n/);
+            return {
+                line: prefix.length,
+                column: prefix.at(-1).length + 1
+            };
+        },
         sourceYaml: {
             value,
             ...redactedPaths.size ? {
@@ -506,6 +540,18 @@ function collectWorkflowDocument(source, options) {
     const sourcePath = source.sourcePath.split(sep).join('/');
     const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
             ...step,
+            meta: {
+                ...step.meta,
+                sourceLocation: snapshot.locate('steps' === phase ? [
+                    'cases',
+                    caseIndex,
+                    'steps',
+                    stepIndex
+                ] : [
+                    phase,
+                    stepIndex
+                ])
+            },
             templateInput: step.input,
             input: resolveWorkflowVariables(step.input, {
                 variables: options.variables,
@@ -680,6 +726,9 @@ const writeTestProjectRunResult = (options)=>{
             markdown: relativeToSummary(result.summaryPath, result.markdownPath)
         } : {},
         summary: result.summary,
+        ...result.metrics ? {
+            metrics: result.metrics
+        } : {},
         projects: result.projects.map((project)=>({
                 projectId: project.projectId,
                 name: project.name,
@@ -1081,7 +1130,8 @@ async function prepareTestRun(options) {
         preflightScope: hasFormatMismatch ? 'run' : 'project',
         reportEnabled: true,
         publications: [],
-        onProgress: options.onProgress ?? (()=>{})
+        onProgress: options.onProgress ?? (()=>{}),
+        onEvent: options.onEvent
     };
 }
 const result_markdown_text = (value)=>value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;').replaceAll('|', '&#124;').replaceAll('[', '&#91;').replaceAll(']', '&#93;').replaceAll('*', '&#42;').replaceAll('_', '&#95;').replaceAll('`', '&#96;').replaceAll('\\', '&#92;').replace(/[\r\n]+/g, ' ');
@@ -1159,6 +1209,38 @@ async function writeTestResultMarkdown(result, index = emptyIndex()) {
     }));
     return output;
 }
+function scopeNodeConsole(node) {
+    return {
+        ...node,
+        execute (ctx) {
+            const position = 'case' === ctx.scope ? ctx.case : ctx.document;
+            const scope = {
+                projectName: position.projectName,
+                sourcePath: position.sourcePath,
+                ...'case' === ctx.scope ? {
+                    caseId: ctx.case.caseId,
+                    caseName: ctx.case.name,
+                    runId: ctx.case.runId
+                } : {
+                    runId: ctx.document.documentRunId
+                },
+                phase: position.phase,
+                stepIndex: position.stepIndex,
+                node: node.name
+            };
+            return runWithConsoleLogScope(scope, ()=>node.execute({
+                    ...ctx,
+                    onTeardown (teardown) {
+                        ctx.onTeardown(()=>runWithConsoleLogScope({
+                                ...scope,
+                                phase: 'teardown',
+                                stepIndex: void 0
+                            }, teardown));
+                    }
+                }));
+        }
+    };
+}
 const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
 const formatStep = (info)=>{
     const position = stepPosition(info);
@@ -1205,69 +1287,121 @@ async function executeDocumentInvocation(options) {
     const documentRetry = 'document' === invocation.retry.scope;
     const attempts = [];
     const fileSetup = invocation.bindings.documentSetup;
-    const documentSetup = options.isProjectResourceFailure && fileSetup ? {
+    const documentSetup = fileSetup ? {
         ...fileSetup,
-        async setup (ctx) {
-            try {
-                return await fileSetup.setup(ctx);
-            } catch (error) {
-                if (isProjectResourceFailure(error)) options.markProjectFatal();
-                throw error;
-            }
+        setup (ctx) {
+            const scope = {
+                projectName: project.name,
+                sourcePath: document.sourcePath,
+                runId: ctx.document.documentRunId,
+                phase: 'fileSetup'
+            };
+            return runWithConsoleLogScope(scope, async ()=>{
+                try {
+                    return await fileSetup.setup({
+                        ...ctx,
+                        onTeardown (teardown) {
+                            ctx.onTeardown(()=>runWithConsoleLogScope({
+                                    ...scope,
+                                    phase: 'teardown'
+                                }, teardown));
+                        }
+                    });
+                } catch (error) {
+                    if (isProjectResourceFailure(error)) options.markProjectFatal();
+                    throw error;
+                }
+            });
         }
-    } : fileSetup;
+    } : void 0;
+    options.onEvent?.({
+        type: 'document-start',
+        documentId: document.documentId,
+        projectName: project.name,
+        sourcePath: document.sourcePath,
+        cases: document.cases.length
+    });
     await runDocumentAttempts({
         retry: documentRetry ? invocation.retry.count : 0,
         signal: options.signal,
         shouldStop: ()=>options.shouldBail() || options.isProjectFatal()
     }, async (documentAttemptIndex)=>{
         if (documentRetry) options.onProgress(`    file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
-        const execution = await runWorkflowDocument(document, {
-            plan: invocation.plan,
-            documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
-            ...invocation.bindings,
-            documentSetup,
-            setupTimeoutMs: options.setupTimeoutMs,
-            teardownTimeoutMs: options.teardownTimeoutMs,
-            project,
-            projectContext: options.projectContext,
-            retry: documentRetry ? 0 : invocation.retry.count,
-            signal: options.signal,
-            defaultTimeoutMs: invocation.defaultTimeoutMs,
-            shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
-            stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
-            isFatalError: (run)=>[
-                    ...run.beforeEach,
-                    ...run.steps,
-                    ...run.afterEach
-                ].some((step)=>step.error && (isFatalDeviceError(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>isFatalDeviceError(error) || isProjectResourceFailure(error)),
-            onCaseStart: (collectedCase)=>{
-                options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
-            },
-            onStepStart: (info)=>{
-                const indent = 'case' === info.scope ? '      ' : '    ';
-                options.onProgress(`${indent}→ ${formatStep(info)}`);
-            },
-            onStepResult: async (info, result)=>{
-                if (isProjectResourceFailure(result.error)) options.markProjectFatal();
-                await invocation.bindings.onStepResult?.(info, result);
-                options.onProgress(formatStepResult(info, result));
-            },
-            onCaseResult: async (attempt)=>{
-                await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
-                options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
-            },
-            onCaseOutcome: (outcome)=>{
-                if (outcome.attempts?.some((attempt)=>[
-                        ...attempt.beforeEach,
-                        ...attempt.steps,
-                        ...attempt.afterEach
-                    ].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
-                if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
-                if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
-            },
-            onDocumentResult: options.onDocumentResult
-        }).catch(async (error)=>{
+        const execution = await runWithConsoleLogScope({
+            projectName: project.name,
+            sourcePath: document.sourcePath
+        }, ()=>runWorkflowDocument(document, {
+                plan: invocation.plan,
+                documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
+                ...invocation.bindings,
+                resolveNode: (name)=>scopeNodeConsole(invocation.bindings.resolveNode(name)),
+                documentSetup,
+                setupTimeoutMs: options.setupTimeoutMs,
+                teardownTimeoutMs: options.teardownTimeoutMs,
+                project,
+                projectContext: options.projectContext,
+                retry: documentRetry ? 0 : invocation.retry.count,
+                signal: options.signal,
+                defaultTimeoutMs: invocation.defaultTimeoutMs,
+                shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
+                stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
+                isFatalError: (run)=>[
+                        ...run.beforeEach,
+                        ...run.steps,
+                        ...run.afterEach
+                    ].some((step)=>step.error && (isFatalDeviceError(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>isFatalDeviceError(error) || isProjectResourceFailure(error)),
+                onCaseStart: (collectedCase)=>{
+                    options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
+                },
+                onStepStart: (info)=>{
+                    options.onEvent?.({
+                        type: 'step-start',
+                        projectName: project.name,
+                        info
+                    });
+                    const indent = 'case' === info.scope ? '      ' : '    ';
+                    options.onProgress(`${indent}→ ${formatStep(info)}`);
+                },
+                onStepResult: async (info, result)=>{
+                    if (isProjectResourceFailure(result.error)) options.markProjectFatal();
+                    await invocation.bindings.onStepResult?.(info, result);
+                    options.onProgress(formatStepResult(info, result));
+                    options.onEvent?.({
+                        type: 'step-end',
+                        projectName: project.name,
+                        info,
+                        result
+                    });
+                },
+                onCaseResult: async (attempt)=>{
+                    await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
+                    options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} case attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
+                    if (!documentRetry) options.onEvent?.({
+                        type: 'case-attempt-end',
+                        attempt,
+                        maxAttempts: invocation.retry.count + 1
+                    });
+                },
+                onCaseOutcome: (outcome)=>{
+                    if (documentRetry) options.onEvent?.({
+                        type: 'case-progress',
+                        outcome,
+                        documentAttemptIndex
+                    });
+                    else options.onEvent?.({
+                        type: 'case-end',
+                        outcome
+                    });
+                    if (outcome.attempts?.some((attempt)=>[
+                            ...attempt.beforeEach,
+                            ...attempt.steps,
+                            ...attempt.afterEach
+                        ].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
+                    if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
+                    if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
+                },
+                onDocumentResult: options.onDocumentResult
+            })).catch(async (error)=>{
             if (error instanceof WorkflowExecutionFailure) {
                 const partial = error.result;
                 const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>isResourceRecoveryError(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
@@ -1337,6 +1471,11 @@ async function executeDocumentInvocation(options) {
             shouldStop: options.shouldBail()
         });
         if (documentRetry) execution.document.retryDecision = decision;
+        if (documentRetry && 'retry' === decision.decision) options.onEvent?.({
+            type: 'document-retry',
+            document: execution.document,
+            maxAttempts: invocation.retry.count + 1
+        });
         attempts.push(execution);
         sinks.cases.push(...execution.cases.map((outcome)=>({
                 ...outcome,
@@ -1371,7 +1510,18 @@ async function executeDocumentInvocation(options) {
     });
     const final = attempts.at(-1);
     if (!final) return;
+    if (documentRetry) for (const outcome of final.cases)options.onEvent?.({
+        type: 'case-end',
+        outcome,
+        documentRetries: attempts.length - 1
+    });
     const failedCases = final.cases.filter((item)=>'failed' === item.status).length;
+    options.onEvent?.({
+        type: 'document-end',
+        document: final.document,
+        cases: final.cases,
+        durationMs: attempts.reduce((total, attempt)=>total + attempt.document.durationMs, 0)
+    });
     if (documentRetry && failedCases) options.addFailedCases(failedCases);
     else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
 }
@@ -1411,11 +1561,45 @@ async function runPreparedFiles(options) {
             code: 'UNRESOLVED_FILE_RESOURCES'
         }));
     };
+    const finalizeProject = (state)=>{
+        const { prepared } = state;
+        const completed = new Set(state.cases.map((item)=>item.caseId));
+        const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
+        for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
+            documentId: document.documentId,
+            caseId: item.caseId,
+            projectName: prepared.project.name,
+            name: item.definition.name,
+            sourcePath: item.sourcePath,
+            caseIndex: item.caseIndex,
+            status: 'not-run',
+            notRunReason: reason
+        });
+        const order = new Map(prepared.documents.map((document, i)=>[
+                document.documentId,
+                i
+            ]));
+        const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
+        state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
+        state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
+    };
     const finishProject = async (state)=>{
         if (!state.runtime || state.finished) return;
         state.finished = true;
+        try {
+            options.onEvent?.({
+                type: 'project-phase',
+                projectName: state.prepared.project.name,
+                phase: 'teardown'
+            });
+        } catch (error) {
+            options.onInfrastructureError(error);
+        }
         const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
-        state.lifecycle = await state.runtime.finish(failed ? 'failed' : 'success');
+        state.lifecycle = await runWithConsoleLogScope({
+            projectName: state.prepared.project.name,
+            phase: 'teardown'
+        }, ()=>state.runtime.finish(failed ? 'failed' : 'success'));
         const releaseProject = ()=>{
             occupiedProjects--;
             wake?.();
@@ -1429,7 +1613,11 @@ async function runPreparedFiles(options) {
                 options.onInfrastructureError(error);
                 releaseProject();
             });
-        } else releaseProject();
+        } else {
+            releaseProject();
+            finalizeProject(state);
+            options.onProjectEnd?.(state);
+        }
     };
     const admit = ()=>{
         for(let offset = 0; offset < states.length; offset++){
@@ -1470,7 +1658,20 @@ async function runPreparedFiles(options) {
                 });
                 setMaxListeners(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
                 options.announceProject(prepared, index);
-                await state.runtime.start();
+                options.onEvent?.({
+                    type: 'project-phase',
+                    projectName: project.name,
+                    phase: 'setup'
+                });
+                await runWithConsoleLogScope({
+                    projectName: project.name,
+                    phase: 'setup'
+                }, ()=>state.runtime.start());
+                options.onEvent?.({
+                    type: 'project-phase',
+                    projectName: project.name,
+                    phase: 'running'
+                });
                 state.initializing = false;
                 if (!state.runtime.canRun) {
                     state.fatal = true;
@@ -1498,8 +1699,9 @@ async function runPreparedFiles(options) {
                     }));
                 },
                 addFailedCases: options.addFailedCases,
-                onProgress: (message)=>options.onProgress(prepared, message),
+                onProgress: (message)=>options.onProgress(prepared, options.maxConcurrentFiles > 1 ? ` [${invocation.document.sourcePath}]${message}` : message),
                 sinks: state,
+                onEvent: options.onEvent,
                 isolatedContext: void 0 !== project.fileSetup,
                 isProjectResourceFailure: isSharedResourceFailure,
                 setupTimeoutMs: options.setupTimeoutMs,
@@ -1578,28 +1780,40 @@ async function runPreparedFiles(options) {
     for (const state of states){
         const { prepared } = state;
         if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
-        const completed = new Set(state.cases.map((item)=>item.caseId));
-        const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
-        for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
-            documentId: document.documentId,
-            caseId: item.caseId,
-            projectName: prepared.project.name,
-            name: item.definition.name,
-            sourcePath: item.sourcePath,
-            caseIndex: item.caseIndex,
-            status: 'not-run',
-            notRunReason: reason
-        });
-        const order = new Map(prepared.documents.map((document, i)=>[
-                document.documentId,
-                i
-            ]));
-        const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
-        state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
-        state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
+        finalizeProject(state);
     }
     return states;
 }
+function configuredModelNames() {
+    const env = globalConfigManager.getAllEnvConfig();
+    const defaultName = env.MIDSCENE_MODEL_NAME;
+    const models = new Map();
+    for (const [intent, name] of [
+        [
+            'default',
+            defaultName
+        ],
+        [
+            'planning',
+            env.MIDSCENE_PLANNING_MODEL_NAME || defaultName
+        ],
+        [
+            'insight',
+            env.MIDSCENE_INSIGHT_MODEL_NAME || defaultName
+        ]
+    ]){
+        if (!name) continue;
+        const intents = models.get(name) ?? [];
+        intents.push(intent);
+        models.set(name, intents);
+    }
+    return [
+        ...models
+    ].map(([name, intents])=>({
+            name,
+            intents
+        }));
+}
 function createOperationJournal(runDir) {
     const path = join(runDir, 'operations.jsonl');
     writeFileSync(path, '', {
@@ -1668,6 +1882,32 @@ async function runPreparedTestProject(plan) {
     const totalDocuments = preparedProjects.reduce((total, prepared)=>total + prepared.documents.length, 0);
     const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
     const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
+    plan.onEvent?.({
+        type: 'run-start',
+        configuredModels: configuredModelNames(),
+        projects: preparedProjects.length,
+        documents: totalDocuments + totalErrors,
+        cases: totalCases,
+        projectRoot,
+        configPath,
+        startedAt: startedAt.toISOString(),
+        maxConcurrentProjects: definition.test.maxConcurrentProjects,
+        maxConcurrentFiles: definition.test.maxConcurrentFiles,
+        files: preparedProjects.flatMap((prepared)=>[
+                ...prepared.documents.map((document)=>({
+                        documentId: document.documentId,
+                        projectName: prepared.project.name,
+                        sourcePath: document.sourcePath,
+                        cases: document.cases.length
+                    })),
+                ...prepared.collectionErrors.map((error)=>({
+                        documentId: error.sourcePath,
+                        projectName: prepared.project.name,
+                        sourcePath: error.sourcePath,
+                        cases: 0
+                    }))
+            ])
+    });
     progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
     const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
     await writeTestRunPlan(runDir, preparedProjects);
@@ -1770,7 +2010,12 @@ async function runPreparedTestProject(plan) {
             shouldBail: bailReached,
             addFailedCases,
             announceProject,
-            onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
+            onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(preparedProjects.length > 1 || effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
+            onEvent: (event)=>runInfrastructureCallback(()=>plan.onEvent?.(event)),
+            onProjectEnd: (state)=>runInfrastructureCallback(()=>plan.onEvent?.({
+                        type: 'project-end',
+                        project: buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle)
+                    })),
             onInfrastructureError: recordInfrastructureError
         });
         states.forEach((state, index)=>{
@@ -1851,6 +2096,10 @@ async function runPreparedTestProject(plan) {
                 buildRunnerDump: (index)=>{
                     reportIndex = index;
                     const dump = buildTestRunReportDump(completedResult, index);
+                    completedResult = {
+                        ...completedResult,
+                        metrics: dump.metrics
+                    };
                     const warn = getDebug('test-runner:report-assembler', {
                         console: true
                     });
@@ -1868,6 +2117,16 @@ async function runPreparedTestProject(plan) {
         await publication.publish(completedResult);
     });
     await publish('write-result', join(dirname(summaryPath), 'result.md'), async ()=>{
+        if (!completedResult.metrics) {
+            const dump = buildTestRunReportDump(completedResult, reportIndex ?? {
+                sources: [],
+                metrics: test_runner_sumTestRunReportMetrics([])
+            });
+            completedResult = {
+                ...completedResult,
+                metrics: dump.metrics
+            };
+        }
         const markdownPath = await writeTestResultMarkdown(completedResult, reportIndex);
         completedResult = {
             ...completedResult,