@midscene/test 1.16.0 → 1.16.2

package/dist/lib/cli/index.js

@@ -28,11 +28,12 @@ __webpack_require__.d(__webpack_exports__, {
     createTestRunId: ()=>createTestRunId,
     runTestProject: ()=>runTestProject,
     discoverTestConfig: ()=>discoverTestConfig,
+    defineFileSetup: ()=>defineFileSetup,
     validateTestFileSelection: ()=>validateTestFileSelection,
     defineProjectSetup: ()=>defineProjectSetup,
     discoverTestFiles: ()=>discoverTestFiles,
-    loadTestProject: ()=>loadTestProject,
-    defineTestProject: ()=>defineTestProject
+    defineTestProject: ()=>defineTestProject,
+    loadTestProject: ()=>loadTestProject
 });
 const external_node_path_namespaceObject = require("node:path");
 const external_node_url_namespaceObject = require("node:url");
@@ -42,6 +43,7 @@ const NodeRegistry = test_runner_namespaceObject.NodeRegistry;
 const projectIdFromIndex = (index)=>`project-${index}`;
 const defineTestProject = (definition)=>definition;
 const defineProjectSetup = (definition)=>definition;
+const defineFileSetup = (definition)=>definition;
 const getDefaultExport = (loaded, absolutePath)=>{
     if ('object' == typeof loaded && null !== loaded && 'default' in loaded) {
         const defaultExport = loaded.default;
@@ -158,9 +160,19 @@ const validateNonNegativeInteger = (value, fallback, label)=>{
     if (!Number.isInteger(value) || value < 0) throw new TypeError(`Midscene config ${label} must be a non-negative integer.`);
     return value;
 };
-const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
+const validateConcurrency = (value, label)=>{
+    if (null === value) return null;
+    return validatePositiveInteger(value, 1, label);
+};
+const validateFileConcurrency = (maxConcurrentFiles, fileSetup, label)=>{
+    if (maxConcurrentFiles > 1 && !fileSetup) throw new TypeError(`Midscene config ${label}.fileSetup is required when the effective maxConcurrentFiles is greater than 1.`);
+};
+const validateExecutionProjects = (value, defaultSetup, defaultFileSetup, globalNodes, test)=>{
     if (void 0 === value) {
         const setup = validateSetup(defaultSetup, 'setup');
+        const fileSetup = validateSetup(defaultFileSetup, 'fileSetup');
+        const maxConcurrentFiles = test.maxConcurrentFiles ?? 1 / 0;
+        validateFileConcurrency(maxConcurrentFiles, fileSetup, 'default');
         return {
             hasExplicitProjects: false,
             projects: Object.freeze([
@@ -170,6 +182,10 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
                     ...setup ? {
                         setup
                     } : {},
+                    ...fileSetup ? {
+                        fileSetup
+                    } : {},
+                    maxConcurrentFiles,
                     tags: Object.freeze({
                         include: [],
                         exclude: []
@@ -182,6 +198,7 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         };
     }
     if (void 0 !== defaultSetup) throw new TypeError('Midscene config setup cannot be used together with projects. Move setup to projects[].setup.');
+    if (void 0 !== defaultFileSetup) throw new TypeError('Midscene config fileSetup cannot be used together with projects. Move fileSetup to projects[].fileSetup.');
     if (!Array.isArray(value) || 0 === value.length) throw new TypeError('Midscene config projects must be a non-empty array when declared.');
     const names = new Set();
     const projects = value.map((candidate, index)=>{
@@ -190,6 +207,8 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         rejectUnknownKeys(candidate, [
             'name',
             'setup',
+            'fileSetup',
+            'maxConcurrentFiles',
             'nodes',
             'files',
             'tags',
@@ -206,10 +225,18 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
             ...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
             ...localNodes.definitions()
         ]);
+        const fileSetup = validateSetup(candidate.fileSetup, `${label}.fileSetup`);
+        const globalFileLimit = test.maxConcurrentFiles ?? 1 / 0;
+        const maxConcurrentFiles = Math.min(globalFileLimit, validatePositiveInteger(candidate.maxConcurrentFiles, globalFileLimit, `${label}.maxConcurrentFiles`));
+        validateFileConcurrency(maxConcurrentFiles, fileSetup, label);
         return Object.freeze({
             projectId: projectIdFromIndex(index),
             name: candidate.name,
             nodes,
+            ...fileSetup ? {
+                fileSetup
+            } : {},
+            maxConcurrentFiles,
             ...files ? {
                 files
             } : {},
@@ -226,11 +253,25 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         hasExplicitProjects: true
     };
 };
-const validateTestOptions = (value)=>{
+const validateTestOptions = (value, hasExplicitProjects)=>{
     if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config test must be an object.');
     const candidate = value ?? {};
+    if (void 0 !== candidate.maxConcurrency) {
+        const limit = 'number' == typeof candidate.maxConcurrency && Number.isInteger(candidate.maxConcurrency) && candidate.maxConcurrency > 0 ? candidate.maxConcurrency : 1;
+        throw new TypeError([
+            'Midscene config test.maxConcurrency is no longer supported.',
+            'It controlled project concurrency, not file concurrency. Remove it and use:',
+            `  test: { maxConcurrentProjects: ${limit}, maxConcurrentFiles: ${hasExplicitProjects ? limit : 1} }`,
+            hasExplicitProjects ? '  Set projects[].maxConcurrentFiles to 1 to keep files sequential within each project.' : '  The implicit default project keeps running files sequentially.',
+            hasExplicitProjects ? 'For parallel files, configure fileSetup and raise both test.maxConcurrentFiles and projects[].maxConcurrentFiles.' : 'For parallel files, configure fileSetup and raise test.maxConcurrentFiles.',
+            'Migration guide: https://midscenejs.com/midscene-test/configuration.html#existing-configurations'
+        ].join('\n'));
+    }
     return Object.freeze({
-        maxConcurrency: validatePositiveInteger(candidate.maxConcurrency, 1, 'test.maxConcurrency'),
+        maxConcurrentFiles: validateConcurrency(candidate.maxConcurrentFiles, 'test.maxConcurrentFiles'),
+        maxConcurrentProjects: validateConcurrency(candidate.maxConcurrentProjects, 'test.maxConcurrentProjects'),
+        setupTimeout: validatePositiveInteger(candidate.setupTimeout, 120000, 'test.setupTimeout'),
+        teardownTimeout: validatePositiveInteger(candidate.teardownTimeout, 30000, 'test.teardownTimeout'),
         bail: validateNonNegativeInteger(candidate.bail, 0, 'test.bail'),
         testTimeout: validatePositiveInteger(candidate.testTimeout, 120000, 'test.testTimeout')
     });
@@ -260,16 +301,18 @@ const validateTestProjectDefinition = (definition)=>{
     if ('testRunner' in definition) throw new TypeError('Midscene config testRunner is not supported. Rename it to test.');
     rejectUnknownKeys(definition, [
         'setup',
+        'fileSetup',
         'projects',
         'test',
         'output',
         'nodes'
     ], 'root');
     const nodes = new NodeRegistry(definition.nodes);
-    const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, nodes);
+    const test = validateTestOptions(definition.test, void 0 !== definition.projects);
+    const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, definition.fileSetup, nodes, test);
     return {
         ...resolvedProjects,
-        test: validateTestOptions(definition.test),
+        test,
         output: validateOutput(definition.output),
         nodes,
         resolveNode: (name)=>nodes.get(name)
@@ -350,11 +393,13 @@ function parseWorkflowSource(source) {
                 start: state.position + sourceOffset,
                 end: 0,
                 value: null,
-                children: []
+                children: [],
+                kind: null
             });
             const node = stack.pop();
             node.end = state.position + sourceOffset;
             node.value = state.result;
+            node.kind = state.kind;
             stack.at(-1)?.children.push(node);
             nodes.push(node);
             if ('mapping' !== state.kind) return;
@@ -384,6 +429,36 @@ function parseWorkflowSource(source) {
     const value = (0, test_runner_namespaceObject.sanitizeReportText)(redacted, 65536);
     return {
         parsed,
+        locate (path) {
+            let node = nodes.at(-1);
+            for (const part of path){
+                if (!node) return;
+                while(1 === node.children.length && node.children[0].value === node.value)node = node.children[0];
+                if ('number' == typeof part && 'sequence' === node.kind) node = node.children[part];
+                else {
+                    if ('string' != typeof part || 'mapping' !== node.kind) return;
+                    const children = node.children;
+                    node = void 0;
+                    for(let index = 0; index < children.length; index += 2)if (children[index].value === part) {
+                        node = children[index + 1];
+                        break;
+                    }
+                }
+            }
+            if (!node) return;
+            let start = node.start;
+            while(start < node.end)if (/\s/.test(source[start])) start++;
+            else if ('#' === source[start]) {
+                const end = source.indexOf('\n', start);
+                if (end < 0) return;
+                start = end + 1;
+            } else break;
+            const prefix = source.slice(0, start).split(/\r\n|\r|\n/);
+            return {
+                line: prefix.length,
+                column: prefix.at(-1).length + 1
+            };
+        },
         sourceYaml: {
             value,
             ...redactedPaths.size ? {
@@ -496,6 +571,18 @@ function collectWorkflowDocument(source, options) {
     const sourcePath = source.sourcePath.split(external_node_path_namespaceObject.sep).join('/');
     const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
             ...step,
+            meta: {
+                ...step.meta,
+                sourceLocation: snapshot.locate('steps' === phase ? [
+                    'cases',
+                    caseIndex,
+                    'steps',
+                    stepIndex
+                ] : [
+                    phase,
+                    stepIndex
+                ])
+            },
             templateInput: step.input,
             input: resolveWorkflowVariables(step.input, {
                 variables: options.variables,
@@ -671,6 +758,9 @@ const writeTestProjectRunResult = (options)=>{
             markdown: relativeToSummary(result.summaryPath, result.markdownPath)
         } : {},
         summary: result.summary,
+        ...result.metrics ? {
+            metrics: result.metrics
+        } : {},
         projects: result.projects.map((project)=>({
                 projectId: project.projectId,
                 name: project.name,
@@ -953,7 +1043,24 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
             defaultTimeoutMs,
             reportEnabled: true,
             bindings: {
-                resolveNode: project.nodes.require.bind(project.nodes)
+                resolveNode: project.nodes.require.bind(project.nodes),
+                ...project.fileSetup ? {
+                    documentSetup: {
+                        name: project.fileSetup.name,
+                        setup ({ projectContext, document, signal, onTeardown }) {
+                            return project.fileSetup.setup({
+                                project,
+                                projectContext,
+                                file: document,
+                                env: Object.freeze({
+                                    ...process.env
+                                }),
+                                signal,
+                                onTeardown
+                            });
+                        }
+                    }
+                } : {}
             }
         });
     } catch (error) {
@@ -1055,7 +1162,8 @@ async function prepareTestRun(options) {
         preflightScope: hasFormatMismatch ? 'run' : 'project',
         reportEnabled: true,
         publications: [],
-        onProgress: options.onProgress ?? (()=>{})
+        onProgress: options.onProgress ?? (()=>{}),
+        onEvent: options.onEvent
     };
 }
 const external_node_events_namespaceObject = require("node:events");
@@ -1136,6 +1244,38 @@ async function writeTestResultMarkdown(result, index = emptyIndex()) {
     }));
     return output;
 }
+function scopeNodeConsole(node) {
+    return {
+        ...node,
+        execute (ctx) {
+            const position = 'case' === ctx.scope ? ctx.case : ctx.document;
+            const scope = {
+                projectName: position.projectName,
+                sourcePath: position.sourcePath,
+                ...'case' === ctx.scope ? {
+                    caseId: ctx.case.caseId,
+                    caseName: ctx.case.name,
+                    runId: ctx.case.runId
+                } : {
+                    runId: ctx.document.documentRunId
+                },
+                phase: position.phase,
+                stepIndex: position.stepIndex,
+                node: node.name
+            };
+            return (0, logger_namespaceObject.runWithConsoleLogScope)(scope, ()=>node.execute({
+                    ...ctx,
+                    onTeardown (teardown) {
+                        ctx.onTeardown(()=>(0, logger_namespaceObject.runWithConsoleLogScope)({
+                                ...scope,
+                                phase: 'teardown',
+                                stepIndex: void 0
+                            }, teardown));
+                    }
+                }));
+        }
+    };
+}
 const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
 const formatStep = (info)=>{
     const position = stepPosition(info);
@@ -1159,59 +1299,149 @@ const documentHasFatalError = (result)=>[
         ...result.beforeAll,
         ...result.afterAll
     ].some((step)=>step.error && (0, test_runner_namespaceObject.isFatalDeviceError)(step.error)) || (result.teardownErrors ?? []).some(test_runner_namespaceObject.isFatalDeviceError) || !!result.executionErrors?.some(test_runner_namespaceObject.isResourceRecoveryError);
+const isSharedResourceFailure = (error)=>{
+    const seen = new Set();
+    const pending = [
+        error
+    ];
+    while(pending.length && seen.size < 50){
+        const item = pending.pop();
+        if ('object' != typeof item || null === item || seen.has(item)) continue;
+        seen.add(item);
+        const failure = item;
+        if ('PROJECT_RESOURCE_ERROR' === failure.code) return true;
+        if (failure.cause) pending.push(failure.cause);
+        if (Array.isArray(failure.errors)) pending.push(...failure.errors.slice(0, 50));
+    }
+    return false;
+};
 async function executeDocumentInvocation(options) {
     const { invocation, project, sinks } = options;
+    const isProjectResourceFailure = options.isProjectResourceFailure ?? (()=>false);
     const { document } = invocation;
     const documentRetry = 'document' === invocation.retry.scope;
     const attempts = [];
+    const fileSetup = invocation.bindings.documentSetup;
+    const documentSetup = fileSetup ? {
+        ...fileSetup,
+        setup (ctx) {
+            const scope = {
+                projectName: project.name,
+                sourcePath: document.sourcePath,
+                runId: ctx.document.documentRunId,
+                phase: 'fileSetup'
+            };
+            return (0, logger_namespaceObject.runWithConsoleLogScope)(scope, async ()=>{
+                try {
+                    return await fileSetup.setup({
+                        ...ctx,
+                        onTeardown (teardown) {
+                            ctx.onTeardown(()=>(0, logger_namespaceObject.runWithConsoleLogScope)({
+                                    ...scope,
+                                    phase: 'teardown'
+                                }, teardown));
+                        }
+                    });
+                } catch (error) {
+                    if (isProjectResourceFailure(error)) options.markProjectFatal();
+                    throw error;
+                }
+            });
+        }
+    } : void 0;
+    options.onEvent?.({
+        type: 'document-start',
+        documentId: document.documentId,
+        projectName: project.name,
+        sourcePath: document.sourcePath,
+        cases: document.cases.length
+    });
     await (0, test_runner_namespaceObject.runDocumentAttempts)({
         retry: documentRetry ? invocation.retry.count : 0,
         signal: options.signal,
         shouldStop: ()=>options.shouldBail() || options.isProjectFatal()
     }, async (documentAttemptIndex)=>{
         if (documentRetry) options.onProgress(`    file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
-        const execution = await (0, test_runner_namespaceObject.runWorkflowDocument)(document, {
-            plan: invocation.plan,
-            documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
-            ...invocation.bindings,
-            project,
-            projectContext: options.projectContext,
-            retry: documentRetry ? 0 : invocation.retry.count,
-            signal: options.signal,
-            defaultTimeoutMs: invocation.defaultTimeoutMs,
-            shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
-            stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
-            isFatalError: (run)=>[
-                    ...run.beforeEach,
-                    ...run.steps,
-                    ...run.afterEach
-                ].some((step)=>step.error && (0, test_runner_namespaceObject.isFatalDeviceError)(step.error)) || (run.teardownErrors ?? []).some(test_runner_namespaceObject.isFatalDeviceError),
-            onCaseStart: (collectedCase)=>{
-                options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
-            },
-            onStepStart: (info)=>{
-                const indent = 'case' === info.scope ? '      ' : '    ';
-                options.onProgress(`${indent}→ ${formatStep(info)}`);
-            },
-            onStepResult: async (info, result)=>{
-                await invocation.bindings.onStepResult?.(info, result);
-                options.onProgress(formatStepResult(info, result));
-            },
-            onCaseResult: async (attempt)=>{
-                await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
-                options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
-            },
-            onCaseOutcome: (outcome)=>{
-                if (caseHasFatalError(outcome)) options.markProjectFatal();
-                if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
-            },
-            onDocumentResult: options.onDocumentResult
-        }).catch(async (error)=>{
+        const execution = await (0, logger_namespaceObject.runWithConsoleLogScope)({
+            projectName: project.name,
+            sourcePath: document.sourcePath
+        }, ()=>(0, test_runner_namespaceObject.runWorkflowDocument)(document, {
+                plan: invocation.plan,
+                documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
+                ...invocation.bindings,
+                resolveNode: (name)=>scopeNodeConsole(invocation.bindings.resolveNode(name)),
+                documentSetup,
+                setupTimeoutMs: options.setupTimeoutMs,
+                teardownTimeoutMs: options.teardownTimeoutMs,
+                project,
+                projectContext: options.projectContext,
+                retry: documentRetry ? 0 : invocation.retry.count,
+                signal: options.signal,
+                defaultTimeoutMs: invocation.defaultTimeoutMs,
+                shouldStop: ()=>options.rootSignal.aborted || options.shouldBail() || options.isProjectFatal(),
+                stopReason: ()=>options.rootSignal.aborted ? 'interrupted' : options.isProjectFatal() ? 'fatal-error' : 'bail',
+                isFatalError: (run)=>[
+                        ...run.beforeEach,
+                        ...run.steps,
+                        ...run.afterEach
+                    ].some((step)=>step.error && ((0, test_runner_namespaceObject.isFatalDeviceError)(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>(0, test_runner_namespaceObject.isFatalDeviceError)(error) || isProjectResourceFailure(error)),
+                onCaseStart: (collectedCase)=>{
+                    options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
+                },
+                onStepStart: (info)=>{
+                    options.onEvent?.({
+                        type: 'step-start',
+                        projectName: project.name,
+                        info
+                    });
+                    const indent = 'case' === info.scope ? '      ' : '    ';
+                    options.onProgress(`${indent}→ ${formatStep(info)}`);
+                },
+                onStepResult: async (info, result)=>{
+                    if (isProjectResourceFailure(result.error)) options.markProjectFatal();
+                    await invocation.bindings.onStepResult?.(info, result);
+                    options.onProgress(formatStepResult(info, result));
+                    options.onEvent?.({
+                        type: 'step-end',
+                        projectName: project.name,
+                        info,
+                        result
+                    });
+                },
+                onCaseResult: async (attempt)=>{
+                    await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
+                    options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} case attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
+                    if (!documentRetry) options.onEvent?.({
+                        type: 'case-attempt-end',
+                        attempt,
+                        maxAttempts: invocation.retry.count + 1
+                    });
+                },
+                onCaseOutcome: (outcome)=>{
+                    if (documentRetry) options.onEvent?.({
+                        type: 'case-progress',
+                        outcome,
+                        documentAttemptIndex
+                    });
+                    else options.onEvent?.({
+                        type: 'case-end',
+                        outcome
+                    });
+                    if (outcome.attempts?.some((attempt)=>[
+                            ...attempt.beforeEach,
+                            ...attempt.steps,
+                            ...attempt.afterEach
+                        ].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
+                    if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
+                    if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
+                },
+                onDocumentResult: options.onDocumentResult
+            })).catch(async (error)=>{
             if (error instanceof test_runner_namespaceObject.WorkflowExecutionFailure) {
                 const partial = error.result;
                 const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>(0, test_runner_namespaceObject.isResourceRecoveryError)(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
                 if (resourceOnly) {
-                    if (error.errors.some(test_runner_namespaceObject.isResourceRecoveryError)) options.markProjectFatal();
+                    if (!options.isolatedContext && error.errors.some(test_runner_namespaceObject.isResourceRecoveryError)) options.markProjectFatal();
                     return partial;
                 }
                 if (documentRetry) partial.document.retryDecision = (0, test_runner_namespaceObject.decideRetry)({
@@ -1240,7 +1470,17 @@ async function executeDocumentInvocation(options) {
             }
             throw error;
         });
-        if (documentHasFatalError(execution.document)) options.markProjectFatal();
+        if (!options.isolatedContext && documentHasFatalError(execution.document)) options.markProjectFatal();
+        if ([
+            ...execution.document.beforeAll,
+            ...execution.document.afterAll,
+            ...(execution.document.hostErrors ?? []).map(({ error })=>({
+                    error
+                })),
+            ...(execution.document.teardownErrors ?? []).map((error)=>({
+                    error
+                }))
+        ].some(({ error })=>isProjectResourceFailure(error))) options.markProjectFatal();
         const owners = [
             execution.document,
             ...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
@@ -1248,7 +1488,10 @@ async function executeDocumentInvocation(options) {
         const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
         const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
         const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
-        if (cleanupPending || cleanupFailed || operationPending) options.markProjectFatal();
+        if (cleanupPending || operationPending) {
+            options.markProjectFatal();
+            options.onUnresolvedResources?.();
+        } else if (cleanupFailed && !options.isolatedContext) options.markProjectFatal();
         const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
         const decision = (0, test_runner_namespaceObject.decideRetry)({
             scope: 'document',
@@ -1263,6 +1506,11 @@ async function executeDocumentInvocation(options) {
             shouldStop: options.shouldBail()
         });
         if (documentRetry) execution.document.retryDecision = decision;
+        if (documentRetry && 'retry' === decision.decision) options.onEvent?.({
+            type: 'document-retry',
+            document: execution.document,
+            maxAttempts: invocation.retry.count + 1
+        });
         attempts.push(execution);
         sinks.cases.push(...execution.cases.map((outcome)=>({
                 ...outcome,
@@ -1297,10 +1545,312 @@ async function executeDocumentInvocation(options) {
     });
     const final = attempts.at(-1);
     if (!final) return;
+    if (documentRetry) for (const outcome of final.cases)options.onEvent?.({
+        type: 'case-end',
+        outcome,
+        documentRetries: attempts.length - 1
+    });
     const failedCases = final.cases.filter((item)=>'failed' === item.status).length;
+    options.onEvent?.({
+        type: 'document-end',
+        document: final.document,
+        cases: final.cases,
+        durationMs: attempts.reduce((total, attempt)=>total + attempt.document.durationMs, 0)
+    });
     if (documentRetry && failedCases) options.addFailedCases(failedCases);
     else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
 }
+async function runPreparedFiles(options) {
+    const states = options.projects.map((prepared)=>({
+            prepared,
+            cases: [],
+            documents: [],
+            cursor: 0,
+            active: 0,
+            initializing: false,
+            finished: false,
+            fatal: false,
+            setupFailed: false
+        }));
+    let nextProject = 0;
+    let resourcesUnresolved = false;
+    let occupied = 0;
+    let occupiedProjects = 0;
+    const cleanupQueue = [];
+    const release = ()=>{
+        occupied--;
+        while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+            occupied++;
+            cleanupQueue.shift()();
+        }
+        wake?.();
+    };
+    let wake;
+    const running = new Set();
+    const stopped = (state)=>options.signal.aborted || options.shouldBail() || state.fatal;
+    const invalid = (state)=>options.preflightFailed || state.prepared.collectionErrors.length > 0;
+    const unresolved = ()=>{
+        if (resourcesUnresolved) return;
+        resourcesUnresolved = true;
+        options.onInfrastructureError(new test_runner_namespaceObject.WorkflowError('Resources are still running or cleanup is pending. No new test files will start.', {
+            code: 'UNRESOLVED_FILE_RESOURCES'
+        }));
+    };
+    const finalizeProject = (state)=>{
+        const { prepared } = state;
+        const completed = new Set(state.cases.map((item)=>item.caseId));
+        const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
+        for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
+            documentId: document.documentId,
+            caseId: item.caseId,
+            projectName: prepared.project.name,
+            name: item.definition.name,
+            sourcePath: item.sourcePath,
+            caseIndex: item.caseIndex,
+            status: 'not-run',
+            notRunReason: reason
+        });
+        const order = new Map(prepared.documents.map((document, i)=>[
+                document.documentId,
+                i
+            ]));
+        const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
+        state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
+        state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
+    };
+    const finishProject = async (state)=>{
+        if (!state.runtime || state.finished) return;
+        state.finished = true;
+        try {
+            options.onEvent?.({
+                type: 'project-phase',
+                projectName: state.prepared.project.name,
+                phase: 'teardown'
+            });
+        } catch (error) {
+            options.onInfrastructureError(error);
+        }
+        const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
+        state.lifecycle = await (0, logger_namespaceObject.runWithConsoleLogScope)({
+            projectName: state.prepared.project.name,
+            phase: 'teardown'
+        }, ()=>state.runtime.finish(failed ? 'failed' : 'success'));
+        const releaseProject = ()=>{
+            occupiedProjects--;
+            wake?.();
+        };
+        if (state.lifecycle.cleanup?.status === 'pending' || state.lifecycle.pendingOperations?.length) {
+            unresolved();
+            Promise.all([
+                (0, test_runner_namespaceObject.waitForScopeOperations)(state.runtime.signal),
+                (0, test_runner_namespaceObject.getResourceCleanupCompletion)(state.runtime.signal)
+            ]).then(releaseProject, (error)=>{
+                options.onInfrastructureError(error);
+                releaseProject();
+            });
+        } else {
+            releaseProject();
+            finalizeProject(state);
+            options.onProjectEnd?.(state);
+        }
+    };
+    const admit = ()=>{
+        for(let offset = 0; offset < states.length; offset++){
+            const index = (nextProject + offset) % states.length;
+            const state = states[index];
+            if (invalid(state) || state.finished || state.initializing) continue;
+            if (!state.runtime && occupiedProjects >= options.maxConcurrentProjects) continue;
+            const cleanupOnly = !!state.runtime && 0 === state.active && (stopped(state) || state.cursor === state.prepared.invocations.length);
+            if (!cleanupOnly && (stopped(state) || state.cursor >= state.prepared.invocations.length || state.active >= (state.prepared.project.maxConcurrentFiles ?? options.maxConcurrentFiles))) continue;
+            nextProject = (index + 1) % states.length;
+            occupied++;
+            state.active++;
+            if (!state.runtime && !cleanupOnly) {
+                occupiedProjects++;
+                state.initializing = true;
+            }
+            const invocationIndex = cleanupOnly ? -1 : state.cursor++;
+            return {
+                state,
+                index,
+                invocationIndex
+            };
+        }
+    };
+    const run = async ({ state, index, invocationIndex })=>{
+        const { prepared } = state;
+        const { project } = prepared;
+        let fileResourcesPending = false;
+        try {
+            if (invocationIndex < 0) return void await finishProject(state);
+            if (!state.runtime) {
+                state.runtime = (0, test_runner_namespaceObject.createProjectRuntime)({
+                    project,
+                    setup: project.setup,
+                    signal: options.signal,
+                    setupTimeoutMs: options.setupTimeoutMs,
+                    teardownTimeoutMs: options.teardownTimeoutMs
+                });
+                (0, external_node_events_namespaceObject.setMaxListeners)(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
+                options.announceProject(prepared, index);
+                options.onEvent?.({
+                    type: 'project-phase',
+                    projectName: project.name,
+                    phase: 'setup'
+                });
+                await (0, logger_namespaceObject.runWithConsoleLogScope)({
+                    projectName: project.name,
+                    phase: 'setup'
+                }, ()=>state.runtime.start());
+                options.onEvent?.({
+                    type: 'project-phase',
+                    projectName: project.name,
+                    phase: 'running'
+                });
+                state.initializing = false;
+                if (!state.runtime.canRun) {
+                    state.fatal = true;
+                    state.setupFailed = true;
+                }
+                wake?.();
+            }
+            if (stopped(state)) return;
+            const invocation = prepared.invocations[invocationIndex];
+            options.onProgress(prepared, `  [document ${invocationIndex + 1}/${prepared.invocations.length}] ${invocation.document.sourcePath}`);
+            await executeDocumentInvocation({
+                invocation,
+                project,
+                projectContext: state.runtime.context,
+                runDir: options.runDir,
+                signal: state.runtime.signal,
+                rootSignal: options.signal,
+                shouldBail: options.shouldBail,
+                isProjectFatal: ()=>state.fatal,
+                markProjectFatal: ()=>{
+                    if (state.fatal) return;
+                    state.fatal = true;
+                    state.runtime?.abort(new test_runner_namespaceObject.WorkflowError(`Project "${project.name}" shared execution environment is unavailable.`, {
+                        code: 'PROJECT_RESOURCE_ERROR'
+                    }));
+                },
+                addFailedCases: options.addFailedCases,
+                onProgress: (message)=>options.onProgress(prepared, options.maxConcurrentFiles > 1 ? ` [${invocation.document.sourcePath}]${message}` : message),
+                sinks: state,
+                onEvent: options.onEvent,
+                isolatedContext: void 0 !== project.fileSetup,
+                isProjectResourceFailure: isSharedResourceFailure,
+                setupTimeoutMs: options.setupTimeoutMs,
+                teardownTimeoutMs: options.teardownTimeoutMs,
+                onUnresolvedResources: ()=>{
+                    fileResourcesPending = true;
+                    unresolved();
+                }
+            });
+        } catch (error) {
+            state.fatal = true;
+            fileResourcesPending = !!state.runtime && (0, test_runner_namespaceObject.getPendingScopeOperations)(state.runtime.signal).length > 0;
+            options.onInfrastructureError(error);
+        } finally{
+            state.initializing = false;
+            try {
+                if (1 === state.active && (stopped(state) || state.cursor === prepared.invocations.length)) await finishProject(state);
+            } catch (error) {
+                options.onInfrastructureError(error);
+            }
+            state.active--;
+            if (state.runtime && (fileResourcesPending || state.lifecycle?.cleanup?.status === 'pending')) (0, test_runner_namespaceObject.waitForScopeOperations)(state.runtime.signal).then(release, (error)=>{
+                options.onInfrastructureError(error);
+                release();
+            });
+            else release();
+            wake?.();
+        }
+    };
+    while(true){
+        const changed = new Promise((resolve)=>{
+            wake = resolve;
+        });
+        while(occupied < options.maxConcurrentFiles){
+            const job = admit();
+            if (!job) break;
+            const task = run(job);
+            running.add(task);
+            const settled = ()=>{
+                running.delete(task);
+                wake?.();
+            };
+            task.then(settled, (error)=>{
+                options.onInfrastructureError(error);
+                settled();
+            });
+        }
+        if (!running.size) break;
+        await changed;
+    }
+    wake = void 0;
+    await Promise.all(states.map(async (state)=>{
+        if (!state.runtime || state.finished) return;
+        const admission = new Promise((resolve)=>{
+            cleanupQueue.push(resolve);
+        });
+        (0, test_runner_namespaceObject.trackResourceOperation)({}, admission, state.runtime.signal);
+        while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+            occupied++;
+            cleanupQueue.shift()();
+        }
+        try {
+            await finishProject(state);
+        } catch (error) {
+            options.onInfrastructureError(error);
+        } finally{
+            Promise.all([
+                admission,
+                (0, test_runner_namespaceObject.getResourceCleanupCompletion)(state.runtime.signal)
+            ]).then(release, (error)=>{
+                options.onInfrastructureError(error);
+                release();
+            });
+        }
+    }));
+    for (const state of states){
+        const { prepared } = state;
+        if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
+        finalizeProject(state);
+    }
+    return states;
+}
+require("node:util");
+const env_namespaceObject = require("@midscene/shared/env");
+function configuredModelNames() {
+    const env = env_namespaceObject.globalConfigManager.getAllEnvConfig();
+    const defaultName = env.MIDSCENE_MODEL_NAME;
+    const models = new Map();
+    for (const [intent, name] of [
+        [
+            'default',
+            defaultName
+        ],
+        [
+            'planning',
+            env.MIDSCENE_PLANNING_MODEL_NAME || defaultName
+        ],
+        [
+            'insight',
+            env.MIDSCENE_INSIGHT_MODEL_NAME || defaultName
+        ]
+    ]){
+        if (!name) continue;
+        const intents = models.get(name) ?? [];
+        intents.push(intent);
+        models.set(name, intents);
+    }
+    return [
+        ...models
+    ].map(([name, intents])=>({
+            name,
+            intents
+        }));
+}
 function createOperationJournal(runDir) {
     const path = (0, external_node_path_namespaceObject.join)(runDir, 'operations.jsonl');
     (0, external_node_fs_namespaceObject.writeFileSync)(path, '', {
@@ -1369,8 +1919,34 @@ async function runPreparedTestProject(plan) {
     const totalDocuments = preparedProjects.reduce((total, prepared)=>total + prepared.documents.length, 0);
     const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
     const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
+    plan.onEvent?.({
+        type: 'run-start',
+        configuredModels: configuredModelNames(),
+        projects: preparedProjects.length,
+        documents: totalDocuments + totalErrors,
+        cases: totalCases,
+        projectRoot,
+        configPath,
+        startedAt: startedAt.toISOString(),
+        maxConcurrentProjects: definition.test.maxConcurrentProjects,
+        maxConcurrentFiles: definition.test.maxConcurrentFiles,
+        files: preparedProjects.flatMap((prepared)=>[
+                ...prepared.documents.map((document)=>({
+                        documentId: document.documentId,
+                        projectName: prepared.project.name,
+                        sourcePath: document.sourcePath,
+                        cases: document.cases.length
+                    })),
+                ...prepared.collectionErrors.map((error)=>({
+                        documentId: error.sourcePath,
+                        projectName: prepared.project.name,
+                        sourcePath: error.sourcePath,
+                        cases: 0
+                    }))
+            ])
+    });
     progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
-    const effectiveConcurrency = Math.min(definition.test.maxConcurrency, preparedProjects.length);
+    const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
     await writeTestRunPlan(runDir, preparedProjects);
     const journal = createOperationJournal(runDir);
     const rootController = new AbortController();
@@ -1384,6 +1960,18 @@ async function runPreparedTestProject(plan) {
     process.on('SIGTERM', sigterm);
     let failedCaseCount = 0;
     const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
+    let shutdownDeadline;
+    const beginShutdown = ()=>{
+        shutdownDeadline ??= Date.now() + definition.test.teardownTimeout;
+    };
+    rootController.signal.addEventListener('abort', beginShutdown, {
+        once: true
+    });
+    const addFailedCases = (count)=>{
+        failedCaseCount += count;
+        if (bailReached()) beginShutdown();
+    };
+    const teardownTimeoutMs = ()=>Math.max(0, Math.min(definition.test.teardownTimeout, void 0 === shutdownDeadline ? 1 / 0 : shutdownDeadline - Date.now()));
     let hasInfrastructureError = false;
     const infrastructureErrors = [];
     const recordInfrastructureError = (error)=>{
@@ -1445,112 +2033,30 @@ async function runPreparedTestProject(plan) {
         announceProject(prepared, projectIndex);
         return buildProjectResult(prepared, notRunSuite(prepared, reason), []);
     };
-    const runPreparedProject = async (prepared, projectIndex)=>{
-        const { project } = prepared;
-        announceProject(prepared, projectIndex);
-        const projectProgress = (message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${project.name}]${message}` : message));
-        const cases = [];
-        const documents = [];
-        let lifecycle;
-        let projectFatal = false;
-        if (prepared.collectionErrors.length > 0 || 'run' === plan.preflightScope && totalErrors > 0) cases.push(...notRunSuite(prepared, 'project-preflight-failed'));
-        else if (rootController.signal.aborted) cases.push(...notRunSuite(prepared, 'interrupted'));
-        else if (bailReached()) cases.push(...notRunSuite(prepared, 'bail'));
-        else {
-            const runtime = (0, test_runner_namespaceObject.createProjectRuntime)({
-                project,
-                setup: project.setup,
-                signal: rootController.signal
-            });
-            let hasProjectExecutionError = false;
-            try {
-                await runtime.start();
-                if (runtime.canRun) {
-                    const runDocument = async (invocation, documentIndex)=>{
-                        const { document } = invocation;
-                        if (rootController.signal.aborted || bailReached() || projectFatal) {
-                            const reason = rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                            cases.push(...document.cases.map((item)=>asNotRun(document.documentId, item, project.name, reason)));
-                            return;
-                        }
-                        projectProgress(`  [document ${documentIndex + 1}/${prepared.documents.length}] ${document.sourcePath}`);
-                        await executeDocumentInvocation({
-                            invocation,
-                            project,
-                            projectContext: runtime.context,
-                            runDir,
-                            signal: runtime.signal,
-                            rootSignal: rootController.signal,
-                            shouldBail: bailReached,
-                            isProjectFatal: ()=>projectFatal,
-                            markProjectFatal: ()=>{
-                                projectFatal = true;
-                            },
-                            addFailedCases: (count)=>{
-                                failedCaseCount += count;
-                            },
-                            onProgress: projectProgress,
-                            sinks: {
-                                cases,
-                                documents
-                            }
-                        });
-                    };
-                    let prerequisiteFailed = false;
-                    const prerequisite = prepared.invocations.find((invocation)=>invocation.document.documentId === prepared.prerequisiteDocumentId);
-                    const prerequisiteCount = prerequisite ? 1 : 0;
-                    if (prerequisite) {
-                        await runDocument(prerequisite, 0);
-                        prerequisiteFailed = documents.at(-1)?.status === 'failed' || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status);
-                    }
-                    if (!prerequisiteFailed) await (0, test_runner_namespaceObject.runConcurrentJobs)(prepared.invocations.filter((invocation)=>invocation !== prerequisite), {
-                        concurrency: prepared.documentConcurrency,
-                        shouldStop: ()=>rootController.signal.aborted || bailReached() || projectFatal
-                    }, async (invocation, index)=>{
-                        try {
-                            await runDocument(invocation, index + prerequisiteCount);
-                        } catch (error) {
-                            recordInfrastructureError(error);
-                            throw error;
-                        }
-                    });
-                    const completed = new Set(cases.map((item)=>item.caseId));
-                    const reason = prerequisiteFailed ? 'project-setup-failed' : rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                    cases.push(...notRunSuite(prepared, reason).filter((item)=>!completed.has(item.caseId)));
-                } else cases.push(...notRunSuite(prepared, rootController.signal.aborted ? 'interrupted' : 'project-setup-failed'));
-            } catch (error) {
-                hasProjectExecutionError = true;
-                recordInfrastructureError(error);
-            } finally{
-                const hasFailure = hasProjectExecutionError || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || projectFatal || rootController.signal.aborted;
-                lifecycle = await runtime.finish(hasFailure ? 'failed' : 'success');
-            }
-            if (hasProjectExecutionError) {
-                const completed = new Set(cases.map((item)=>item.caseId));
-                cases.push(...notRunSuite(prepared, 'interrupted').filter((item)=>!completed.has(item.caseId)));
-            }
-        }
-        const order = new Map(prepared.documents.map((document, index)=>[
-                document.documentId,
-                index
-            ]));
-        const documentOrder = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
-        cases.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || a.caseIndex - b.caseIndex);
-        documents.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || (a.attemptIndex ?? 0) - (b.attemptIndex ?? 0));
-        return buildProjectResult(prepared, cases, documents, lifecycle);
-    };
     const projectResults = new Array(preparedProjects.length);
     try {
-        await (0, test_runner_namespaceObject.runConcurrentJobs)(preparedProjects, {
-            concurrency: effectiveConcurrency,
-            shouldStop: ()=>hasInfrastructureError || rootController.signal.aborted || bailReached()
-        }, async (prepared, index)=>{
-            try {
-                projectResults[index] = await runPreparedProject(prepared, index);
-            } catch (error) {
-                recordInfrastructureError(error);
-                throw error;
-            }
+        const states = await runPreparedFiles({
+            projects: preparedProjects,
+            maxConcurrentFiles: effectiveConcurrency,
+            maxConcurrentProjects: definition.test.maxConcurrentProjects ?? 1 / 0,
+            setupTimeoutMs: definition.test.setupTimeout,
+            teardownTimeoutMs,
+            signal: rootController.signal,
+            runDir,
+            preflightFailed: 'run' === plan.preflightScope && totalErrors > 0,
+            shouldBail: bailReached,
+            addFailedCases,
+            announceProject,
+            onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(preparedProjects.length > 1 || effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
+            onEvent: (event)=>runInfrastructureCallback(()=>plan.onEvent?.(event)),
+            onProjectEnd: (state)=>runInfrastructureCallback(()=>plan.onEvent?.({
+                        type: 'project-end',
+                        project: buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle)
+                    })),
+            onInfrastructureError: recordInfrastructureError
+        });
+        states.forEach((state, index)=>{
+            projectResults[index] = buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle);
         });
     } catch (error) {
         recordInfrastructureError(error);
@@ -1627,6 +2133,10 @@ async function runPreparedTestProject(plan) {
                 buildRunnerDump: (index)=>{
                     reportIndex = index;
                     const dump = (0, test_runner_namespaceObject.buildTestRunReportDump)(completedResult, index);
+                    completedResult = {
+                        ...completedResult,
+                        metrics: dump.metrics
+                    };
                     const warn = (0, logger_namespaceObject.getDebug)('test-runner:report-assembler', {
                         console: true
                     });
@@ -1644,6 +2154,16 @@ async function runPreparedTestProject(plan) {
         await publication.publish(completedResult);
     });
     await publish('write-result', (0, external_node_path_namespaceObject.join)((0, external_node_path_namespaceObject.dirname)(summaryPath), 'result.md'), async ()=>{
+        if (!completedResult.metrics) {
+            const dump = (0, test_runner_namespaceObject.buildTestRunReportDump)(completedResult, reportIndex ?? {
+                sources: [],
+                metrics: (0, test_runner_namespaceObject.sumTestRunReportMetrics)([])
+            });
+            completedResult = {
+                ...completedResult,
+                metrics: dump.metrics
+            };
+        }
         const markdownPath = await writeTestResultMarkdown(completedResult, reportIndex);
         completedResult = {
             ...completedResult,
@@ -1659,6 +2179,7 @@ async function runTestProject(options = {}) {
 }
 exports.DEFAULT_TEST_FILE_SELECTION = __webpack_exports__.DEFAULT_TEST_FILE_SELECTION;
 exports.createTestRunId = __webpack_exports__.createTestRunId;
+exports.defineFileSetup = __webpack_exports__.defineFileSetup;
 exports.defineProjectSetup = __webpack_exports__.defineProjectSetup;
 exports.defineTestProject = __webpack_exports__.defineTestProject;
 exports.discoverTestConfig = __webpack_exports__.discoverTestConfig;
@@ -1669,6 +2190,7 @@ exports.validateTestFileSelection = __webpack_exports__.validateTestFileSelectio
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "DEFAULT_TEST_FILE_SELECTION",
     "createTestRunId",
+    "defineFileSetup",
     "defineProjectSetup",
     "defineTestProject",
     "discoverTestConfig",