@midscene/test 1.16.0 → 1.16.1

package/dist/es/test-cli.mjs

@@ -1,6 +1,6 @@
 import * as __rspack_external_node_fs_5ea92f0c from "node:fs";
 import * as __rspack_external_node_path_c5b9b54f from "node:path";
-import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText } from "@midscene/core/internal/test-runner";
+import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, getPendingScopeOperations, getResourceCleanupCompletion, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText, trackResourceOperation, waitForScopeOperations } from "@midscene/core/internal/test-runner";
 import { inspect } from "node:util";
 import { z } from "zod/v4";
 import { pathToFileURL } from "node:url";
@@ -21,7 +21,7 @@ var __webpack_modules__ = {
     },
     "./package.json" (module) {
         module.exports = {
-            rE: "1.16.0"
+            rE: "1.16.1"
         };
     }
 };
@@ -352,9 +352,19 @@ const validateNonNegativeInteger = (value, fallback, label)=>{
     if (!Number.isInteger(value) || value < 0) throw new TypeError(`Midscene config ${label} must be a non-negative integer.`);
     return value;
 };
-const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
+const validateConcurrency = (value, label)=>{
+    if (null === value) return null;
+    return validatePositiveInteger(value, 1, label);
+};
+const validateFileConcurrency = (maxConcurrentFiles, fileSetup, label)=>{
+    if (maxConcurrentFiles > 1 && !fileSetup) throw new TypeError(`Midscene config ${label}.fileSetup is required when the effective maxConcurrentFiles is greater than 1.`);
+};
+const validateExecutionProjects = (value, defaultSetup, defaultFileSetup, globalNodes, test)=>{
     if (void 0 === value) {
         const setup = validateSetup(defaultSetup, 'setup');
+        const fileSetup = validateSetup(defaultFileSetup, 'fileSetup');
+        const maxConcurrentFiles = test.maxConcurrentFiles ?? 1 / 0;
+        validateFileConcurrency(maxConcurrentFiles, fileSetup, 'default');
         return {
             hasExplicitProjects: false,
             projects: Object.freeze([
@@ -364,6 +374,10 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
                     ...setup ? {
                         setup
                     } : {},
+                    ...fileSetup ? {
+                        fileSetup
+                    } : {},
+                    maxConcurrentFiles,
                     tags: Object.freeze({
                         include: [],
                         exclude: []
@@ -376,6 +390,7 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         };
     }
     if (void 0 !== defaultSetup) throw new TypeError('Midscene config setup cannot be used together with projects. Move setup to projects[].setup.');
+    if (void 0 !== defaultFileSetup) throw new TypeError('Midscene config fileSetup cannot be used together with projects. Move fileSetup to projects[].fileSetup.');
     if (!Array.isArray(value) || 0 === value.length) throw new TypeError('Midscene config projects must be a non-empty array when declared.');
     const names = new Set();
     const projects = value.map((candidate, index)=>{
@@ -384,6 +399,8 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         rejectUnknownKeys(candidate, [
             'name',
             'setup',
+            'fileSetup',
+            'maxConcurrentFiles',
             'nodes',
             'files',
             'tags',
@@ -400,10 +417,18 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
             ...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
             ...localNodes.definitions()
         ]);
+        const fileSetup = validateSetup(candidate.fileSetup, `${label}.fileSetup`);
+        const globalFileLimit = test.maxConcurrentFiles ?? 1 / 0;
+        const maxConcurrentFiles = Math.min(globalFileLimit, validatePositiveInteger(candidate.maxConcurrentFiles, globalFileLimit, `${label}.maxConcurrentFiles`));
+        validateFileConcurrency(maxConcurrentFiles, fileSetup, label);
         return Object.freeze({
             projectId: projectIdFromIndex(index),
             name: candidate.name,
             nodes,
+            ...fileSetup ? {
+                fileSetup
+            } : {},
+            maxConcurrentFiles,
             ...files ? {
                 files
             } : {},
@@ -420,11 +445,25 @@ const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
         hasExplicitProjects: true
     };
 };
-const validateTestOptions = (value)=>{
+const validateTestOptions = (value, hasExplicitProjects)=>{
     if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config test must be an object.');
     const candidate = value ?? {};
+    if (void 0 !== candidate.maxConcurrency) {
+        const limit = 'number' == typeof candidate.maxConcurrency && Number.isInteger(candidate.maxConcurrency) && candidate.maxConcurrency > 0 ? candidate.maxConcurrency : 1;
+        throw new TypeError([
+            'Midscene config test.maxConcurrency is no longer supported.',
+            'It controlled project concurrency, not file concurrency. Remove it and use:',
+            `  test: { maxConcurrentProjects: ${limit}, maxConcurrentFiles: ${hasExplicitProjects ? limit : 1} }`,
+            hasExplicitProjects ? '  Set projects[].maxConcurrentFiles to 1 to keep files sequential within each project.' : '  The implicit default project keeps running files sequentially.',
+            hasExplicitProjects ? 'For parallel files, configure fileSetup and raise both test.maxConcurrentFiles and projects[].maxConcurrentFiles.' : 'For parallel files, configure fileSetup and raise test.maxConcurrentFiles.',
+            'Migration guide: https://midscenejs.com/midscene-test/configuration.html#existing-configurations'
+        ].join('\n'));
+    }
     return Object.freeze({
-        maxConcurrency: validatePositiveInteger(candidate.maxConcurrency, 1, 'test.maxConcurrency'),
+        maxConcurrentFiles: validateConcurrency(candidate.maxConcurrentFiles, 'test.maxConcurrentFiles'),
+        maxConcurrentProjects: validateConcurrency(candidate.maxConcurrentProjects, 'test.maxConcurrentProjects'),
+        setupTimeout: validatePositiveInteger(candidate.setupTimeout, 120000, 'test.setupTimeout'),
+        teardownTimeout: validatePositiveInteger(candidate.teardownTimeout, 30000, 'test.teardownTimeout'),
         bail: validateNonNegativeInteger(candidate.bail, 0, 'test.bail'),
         testTimeout: validatePositiveInteger(candidate.testTimeout, 120000, 'test.testTimeout')
     });
@@ -454,16 +493,18 @@ const validateTestProjectDefinition = (definition)=>{
     if ('testRunner' in definition) throw new TypeError('Midscene config testRunner is not supported. Rename it to test.');
     rejectUnknownKeys(definition, [
         'setup',
+        'fileSetup',
         'projects',
         'test',
         'output',
         'nodes'
     ], 'root');
     const nodes = new registry_NodeRegistry(definition.nodes);
-    const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, nodes);
+    const test = validateTestOptions(definition.test, void 0 !== definition.projects);
+    const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, definition.fileSetup, nodes, test);
     return {
         ...resolvedProjects,
-        test: validateTestOptions(definition.test),
+        test,
         output: validateOutput(definition.output),
         nodes,
         resolveNode: (name)=>nodes.get(name)
@@ -1137,7 +1178,24 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
             defaultTimeoutMs,
             reportEnabled: true,
             bindings: {
-                resolveNode: project.nodes.require.bind(project.nodes)
+                resolveNode: project.nodes.require.bind(project.nodes),
+                ...project.fileSetup ? {
+                    documentSetup: {
+                        name: project.fileSetup.name,
+                        setup ({ projectContext, document, signal, onTeardown }) {
+                            return project.fileSetup.setup({
+                                project,
+                                projectContext,
+                                file: document,
+                                env: Object.freeze({
+                                    ...process.env
+                                }),
+                                signal,
+                                onTeardown
+                            });
+                        }
+                    }
+                } : {}
             }
         });
     } catch (error) {
@@ -1340,11 +1398,40 @@ const documentHasFatalError = (result)=>[
         ...result.beforeAll,
         ...result.afterAll
     ].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError) || !!result.executionErrors?.some(isResourceRecoveryError);
+const isSharedResourceFailure = (error)=>{
+    const seen = new Set();
+    const pending = [
+        error
+    ];
+    while(pending.length && seen.size < 50){
+        const item = pending.pop();
+        if ('object' != typeof item || null === item || seen.has(item)) continue;
+        seen.add(item);
+        const failure = item;
+        if ('PROJECT_RESOURCE_ERROR' === failure.code) return true;
+        if (failure.cause) pending.push(failure.cause);
+        if (Array.isArray(failure.errors)) pending.push(...failure.errors.slice(0, 50));
+    }
+    return false;
+};
 async function executeDocumentInvocation(options) {
     const { invocation, project, sinks } = options;
+    const isProjectResourceFailure = options.isProjectResourceFailure ?? (()=>false);
     const { document } = invocation;
     const documentRetry = 'document' === invocation.retry.scope;
     const attempts = [];
+    const fileSetup = invocation.bindings.documentSetup;
+    const documentSetup = options.isProjectResourceFailure && fileSetup ? {
+        ...fileSetup,
+        async setup (ctx) {
+            try {
+                return await fileSetup.setup(ctx);
+            } catch (error) {
+                if (isProjectResourceFailure(error)) options.markProjectFatal();
+                throw error;
+            }
+        }
+    } : fileSetup;
     await runDocumentAttempts({
         retry: documentRetry ? invocation.retry.count : 0,
         signal: options.signal,
@@ -1355,6 +1442,9 @@ async function executeDocumentInvocation(options) {
             plan: invocation.plan,
             documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
             ...invocation.bindings,
+            documentSetup,
+            setupTimeoutMs: options.setupTimeoutMs,
+            teardownTimeoutMs: options.teardownTimeoutMs,
             project,
             projectContext: options.projectContext,
             retry: documentRetry ? 0 : invocation.retry.count,
@@ -1366,7 +1456,7 @@ async function executeDocumentInvocation(options) {
                     ...run.beforeEach,
                     ...run.steps,
                     ...run.afterEach
-                ].some((step)=>step.error && isFatalDeviceError(step.error)) || (run.teardownErrors ?? []).some(isFatalDeviceError),
+                ].some((step)=>step.error && (isFatalDeviceError(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>isFatalDeviceError(error) || isProjectResourceFailure(error)),
             onCaseStart: (collectedCase)=>{
                 options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
             },
@@ -1375,6 +1465,7 @@ async function executeDocumentInvocation(options) {
                 options.onProgress(`${indent}→ ${formatStep(info)}`);
             },
             onStepResult: async (info, result)=>{
+                if (isProjectResourceFailure(result.error)) options.markProjectFatal();
                 await invocation.bindings.onStepResult?.(info, result);
                 options.onProgress(formatStepResult(info, result));
             },
@@ -1383,7 +1474,12 @@ async function executeDocumentInvocation(options) {
                 options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
             },
             onCaseOutcome: (outcome)=>{
-                if (caseHasFatalError(outcome)) options.markProjectFatal();
+                if (outcome.attempts?.some((attempt)=>[
+                        ...attempt.beforeEach,
+                        ...attempt.steps,
+                        ...attempt.afterEach
+                    ].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
+                if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
                 if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
             },
             onDocumentResult: options.onDocumentResult
@@ -1392,7 +1488,7 @@ async function executeDocumentInvocation(options) {
                 const partial = error.result;
                 const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>isResourceRecoveryError(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
                 if (resourceOnly) {
-                    if (error.errors.some(isResourceRecoveryError)) options.markProjectFatal();
+                    if (!options.isolatedContext && error.errors.some(isResourceRecoveryError)) options.markProjectFatal();
                     return partial;
                 }
                 if (documentRetry) partial.document.retryDecision = decideRetry({
@@ -1421,7 +1517,17 @@ async function executeDocumentInvocation(options) {
             }
             throw error;
         });
-        if (documentHasFatalError(execution.document)) options.markProjectFatal();
+        if (!options.isolatedContext && documentHasFatalError(execution.document)) options.markProjectFatal();
+        if ([
+            ...execution.document.beforeAll,
+            ...execution.document.afterAll,
+            ...(execution.document.hostErrors ?? []).map(({ error })=>({
+                    error
+                })),
+            ...(execution.document.teardownErrors ?? []).map((error)=>({
+                    error
+                }))
+        ].some(({ error })=>isProjectResourceFailure(error))) options.markProjectFatal();
         const owners = [
             execution.document,
             ...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
@@ -1429,7 +1535,10 @@ async function executeDocumentInvocation(options) {
         const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
         const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
         const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
-        if (cleanupPending || cleanupFailed || operationPending) options.markProjectFatal();
+        if (cleanupPending || operationPending) {
+            options.markProjectFatal();
+            options.onUnresolvedResources?.();
+        } else if (cleanupFailed && !options.isolatedContext) options.markProjectFatal();
         const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
         const decision = decideRetry({
             scope: 'document',
@@ -1482,6 +1591,231 @@ async function executeDocumentInvocation(options) {
     if (documentRetry && failedCases) options.addFailedCases(failedCases);
     else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
 }
+async function runPreparedFiles(options) {
+    const states = options.projects.map((prepared)=>({
+            prepared,
+            cases: [],
+            documents: [],
+            cursor: 0,
+            active: 0,
+            initializing: false,
+            finished: false,
+            fatal: false,
+            setupFailed: false
+        }));
+    let nextProject = 0;
+    let resourcesUnresolved = false;
+    let occupied = 0;
+    let occupiedProjects = 0;
+    const cleanupQueue = [];
+    const release = ()=>{
+        occupied--;
+        while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+            occupied++;
+            cleanupQueue.shift()();
+        }
+        wake?.();
+    };
+    let wake;
+    const running = new Set();
+    const stopped = (state)=>options.signal.aborted || options.shouldBail() || state.fatal;
+    const invalid = (state)=>options.preflightFailed || state.prepared.collectionErrors.length > 0;
+    const unresolved = ()=>{
+        if (resourcesUnresolved) return;
+        resourcesUnresolved = true;
+        options.onInfrastructureError(new WorkflowError('Resources are still running or cleanup is pending. No new test files will start.', {
+            code: 'UNRESOLVED_FILE_RESOURCES'
+        }));
+    };
+    const finishProject = async (state)=>{
+        if (!state.runtime || state.finished) return;
+        state.finished = true;
+        const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
+        state.lifecycle = await state.runtime.finish(failed ? 'failed' : 'success');
+        const releaseProject = ()=>{
+            occupiedProjects--;
+            wake?.();
+        };
+        if (state.lifecycle.cleanup?.status === 'pending' || state.lifecycle.pendingOperations?.length) {
+            unresolved();
+            Promise.all([
+                waitForScopeOperations(state.runtime.signal),
+                getResourceCleanupCompletion(state.runtime.signal)
+            ]).then(releaseProject, (error)=>{
+                options.onInfrastructureError(error);
+                releaseProject();
+            });
+        } else releaseProject();
+    };
+    const admit = ()=>{
+        for(let offset = 0; offset < states.length; offset++){
+            const index = (nextProject + offset) % states.length;
+            const state = states[index];
+            if (invalid(state) || state.finished || state.initializing) continue;
+            if (!state.runtime && occupiedProjects >= options.maxConcurrentProjects) continue;
+            const cleanupOnly = !!state.runtime && 0 === state.active && (stopped(state) || state.cursor === state.prepared.invocations.length);
+            if (!cleanupOnly && (stopped(state) || state.cursor >= state.prepared.invocations.length || state.active >= (state.prepared.project.maxConcurrentFiles ?? options.maxConcurrentFiles))) continue;
+            nextProject = (index + 1) % states.length;
+            occupied++;
+            state.active++;
+            if (!state.runtime && !cleanupOnly) {
+                occupiedProjects++;
+                state.initializing = true;
+            }
+            const invocationIndex = cleanupOnly ? -1 : state.cursor++;
+            return {
+                state,
+                index,
+                invocationIndex
+            };
+        }
+    };
+    const run = async ({ state, index, invocationIndex })=>{
+        const { prepared } = state;
+        const { project } = prepared;
+        let fileResourcesPending = false;
+        try {
+            if (invocationIndex < 0) return void await finishProject(state);
+            if (!state.runtime) {
+                state.runtime = createProjectRuntime({
+                    project,
+                    setup: project.setup,
+                    signal: options.signal,
+                    setupTimeoutMs: options.setupTimeoutMs,
+                    teardownTimeoutMs: options.teardownTimeoutMs
+                });
+                setMaxListeners(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
+                options.announceProject(prepared, index);
+                await state.runtime.start();
+                state.initializing = false;
+                if (!state.runtime.canRun) {
+                    state.fatal = true;
+                    state.setupFailed = true;
+                }
+                wake?.();
+            }
+            if (stopped(state)) return;
+            const invocation = prepared.invocations[invocationIndex];
+            options.onProgress(prepared, `  [document ${invocationIndex + 1}/${prepared.invocations.length}] ${invocation.document.sourcePath}`);
+            await executeDocumentInvocation({
+                invocation,
+                project,
+                projectContext: state.runtime.context,
+                runDir: options.runDir,
+                signal: state.runtime.signal,
+                rootSignal: options.signal,
+                shouldBail: options.shouldBail,
+                isProjectFatal: ()=>state.fatal,
+                markProjectFatal: ()=>{
+                    if (state.fatal) return;
+                    state.fatal = true;
+                    state.runtime?.abort(new WorkflowError(`Project "${project.name}" shared execution environment is unavailable.`, {
+                        code: 'PROJECT_RESOURCE_ERROR'
+                    }));
+                },
+                addFailedCases: options.addFailedCases,
+                onProgress: (message)=>options.onProgress(prepared, message),
+                sinks: state,
+                isolatedContext: void 0 !== project.fileSetup,
+                isProjectResourceFailure: isSharedResourceFailure,
+                setupTimeoutMs: options.setupTimeoutMs,
+                teardownTimeoutMs: options.teardownTimeoutMs,
+                onUnresolvedResources: ()=>{
+                    fileResourcesPending = true;
+                    unresolved();
+                }
+            });
+        } catch (error) {
+            state.fatal = true;
+            fileResourcesPending = !!state.runtime && getPendingScopeOperations(state.runtime.signal).length > 0;
+            options.onInfrastructureError(error);
+        } finally{
+            state.initializing = false;
+            try {
+                if (1 === state.active && (stopped(state) || state.cursor === prepared.invocations.length)) await finishProject(state);
+            } catch (error) {
+                options.onInfrastructureError(error);
+            }
+            state.active--;
+            if (state.runtime && (fileResourcesPending || state.lifecycle?.cleanup?.status === 'pending')) waitForScopeOperations(state.runtime.signal).then(release, (error)=>{
+                options.onInfrastructureError(error);
+                release();
+            });
+            else release();
+            wake?.();
+        }
+    };
+    while(true){
+        const changed = new Promise((resolve)=>{
+            wake = resolve;
+        });
+        while(occupied < options.maxConcurrentFiles){
+            const job = admit();
+            if (!job) break;
+            const task = run(job);
+            running.add(task);
+            const settled = ()=>{
+                running.delete(task);
+                wake?.();
+            };
+            task.then(settled, (error)=>{
+                options.onInfrastructureError(error);
+                settled();
+            });
+        }
+        if (!running.size) break;
+        await changed;
+    }
+    wake = void 0;
+    await Promise.all(states.map(async (state)=>{
+        if (!state.runtime || state.finished) return;
+        const admission = new Promise((resolve)=>{
+            cleanupQueue.push(resolve);
+        });
+        trackResourceOperation({}, admission, state.runtime.signal);
+        while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+            occupied++;
+            cleanupQueue.shift()();
+        }
+        try {
+            await finishProject(state);
+        } catch (error) {
+            options.onInfrastructureError(error);
+        } finally{
+            Promise.all([
+                admission,
+                getResourceCleanupCompletion(state.runtime.signal)
+            ]).then(release, (error)=>{
+                options.onInfrastructureError(error);
+                release();
+            });
+        }
+    }));
+    for (const state of states){
+        const { prepared } = state;
+        if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
+        const completed = new Set(state.cases.map((item)=>item.caseId));
+        const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
+        for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
+            documentId: document.documentId,
+            caseId: item.caseId,
+            projectName: prepared.project.name,
+            name: item.definition.name,
+            sourcePath: item.sourcePath,
+            caseIndex: item.caseIndex,
+            status: 'not-run',
+            notRunReason: reason
+        });
+        const order = new Map(prepared.documents.map((document, i)=>[
+                document.documentId,
+                i
+            ]));
+        const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
+        state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
+        state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
+    }
+    return states;
+}
 function createOperationJournal(runDir) {
     const path = (0, external_node_path_.join)(runDir, 'operations.jsonl');
     (0, external_node_fs_.writeFileSync)(path, '', {
@@ -1551,7 +1885,7 @@ async function runPreparedTestProject(plan) {
     const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
     const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
     progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
-    const effectiveConcurrency = Math.min(definition.test.maxConcurrency, preparedProjects.length);
+    const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
     await writeTestRunPlan(runDir, preparedProjects);
     const journal = createOperationJournal(runDir);
     const rootController = new AbortController();
@@ -1565,6 +1899,18 @@ async function runPreparedTestProject(plan) {
     process.on('SIGTERM', sigterm);
     let failedCaseCount = 0;
     const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
+    let shutdownDeadline;
+    const beginShutdown = ()=>{
+        shutdownDeadline ??= Date.now() + definition.test.teardownTimeout;
+    };
+    rootController.signal.addEventListener('abort', beginShutdown, {
+        once: true
+    });
+    const addFailedCases = (count)=>{
+        failedCaseCount += count;
+        if (bailReached()) beginShutdown();
+    };
+    const teardownTimeoutMs = ()=>Math.max(0, Math.min(definition.test.teardownTimeout, void 0 === shutdownDeadline ? 1 / 0 : shutdownDeadline - Date.now()));
     let hasInfrastructureError = false;
     const infrastructureErrors = [];
     const recordInfrastructureError = (error)=>{
@@ -1626,112 +1972,25 @@ async function runPreparedTestProject(plan) {
         announceProject(prepared, projectIndex);
         return buildProjectResult(prepared, notRunSuite(prepared, reason), []);
     };
-    const runPreparedProject = async (prepared, projectIndex)=>{
-        const { project } = prepared;
-        announceProject(prepared, projectIndex);
-        const projectProgress = (message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${project.name}]${message}` : message));
-        const cases = [];
-        const documents = [];
-        let lifecycle;
-        let projectFatal = false;
-        if (prepared.collectionErrors.length > 0 || 'run' === plan.preflightScope && totalErrors > 0) cases.push(...notRunSuite(prepared, 'project-preflight-failed'));
-        else if (rootController.signal.aborted) cases.push(...notRunSuite(prepared, 'interrupted'));
-        else if (bailReached()) cases.push(...notRunSuite(prepared, 'bail'));
-        else {
-            const runtime = createProjectRuntime({
-                project,
-                setup: project.setup,
-                signal: rootController.signal
-            });
-            let hasProjectExecutionError = false;
-            try {
-                await runtime.start();
-                if (runtime.canRun) {
-                    const runDocument = async (invocation, documentIndex)=>{
-                        const { document } = invocation;
-                        if (rootController.signal.aborted || bailReached() || projectFatal) {
-                            const reason = rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                            cases.push(...document.cases.map((item)=>asNotRun(document.documentId, item, project.name, reason)));
-                            return;
-                        }
-                        projectProgress(`  [document ${documentIndex + 1}/${prepared.documents.length}] ${document.sourcePath}`);
-                        await executeDocumentInvocation({
-                            invocation,
-                            project,
-                            projectContext: runtime.context,
-                            runDir,
-                            signal: runtime.signal,
-                            rootSignal: rootController.signal,
-                            shouldBail: bailReached,
-                            isProjectFatal: ()=>projectFatal,
-                            markProjectFatal: ()=>{
-                                projectFatal = true;
-                            },
-                            addFailedCases: (count)=>{
-                                failedCaseCount += count;
-                            },
-                            onProgress: projectProgress,
-                            sinks: {
-                                cases,
-                                documents
-                            }
-                        });
-                    };
-                    let prerequisiteFailed = false;
-                    const prerequisite = prepared.invocations.find((invocation)=>invocation.document.documentId === prepared.prerequisiteDocumentId);
-                    const prerequisiteCount = prerequisite ? 1 : 0;
-                    if (prerequisite) {
-                        await runDocument(prerequisite, 0);
-                        prerequisiteFailed = documents.at(-1)?.status === 'failed' || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status);
-                    }
-                    if (!prerequisiteFailed) await runConcurrentJobs(prepared.invocations.filter((invocation)=>invocation !== prerequisite), {
-                        concurrency: prepared.documentConcurrency,
-                        shouldStop: ()=>rootController.signal.aborted || bailReached() || projectFatal
-                    }, async (invocation, index)=>{
-                        try {
-                            await runDocument(invocation, index + prerequisiteCount);
-                        } catch (error) {
-                            recordInfrastructureError(error);
-                            throw error;
-                        }
-                    });
-                    const completed = new Set(cases.map((item)=>item.caseId));
-                    const reason = prerequisiteFailed ? 'project-setup-failed' : rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                    cases.push(...notRunSuite(prepared, reason).filter((item)=>!completed.has(item.caseId)));
-                } else cases.push(...notRunSuite(prepared, rootController.signal.aborted ? 'interrupted' : 'project-setup-failed'));
-            } catch (error) {
-                hasProjectExecutionError = true;
-                recordInfrastructureError(error);
-            } finally{
-                const hasFailure = hasProjectExecutionError || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || projectFatal || rootController.signal.aborted;
-                lifecycle = await runtime.finish(hasFailure ? 'failed' : 'success');
-            }
-            if (hasProjectExecutionError) {
-                const completed = new Set(cases.map((item)=>item.caseId));
-                cases.push(...notRunSuite(prepared, 'interrupted').filter((item)=>!completed.has(item.caseId)));
-            }
-        }
-        const order = new Map(prepared.documents.map((document, index)=>[
-                document.documentId,
-                index
-            ]));
-        const documentOrder = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
-        cases.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || a.caseIndex - b.caseIndex);
-        documents.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || (a.attemptIndex ?? 0) - (b.attemptIndex ?? 0));
-        return buildProjectResult(prepared, cases, documents, lifecycle);
-    };
     const projectResults = new Array(preparedProjects.length);
     try {
-        await runConcurrentJobs(preparedProjects, {
-            concurrency: effectiveConcurrency,
-            shouldStop: ()=>hasInfrastructureError || rootController.signal.aborted || bailReached()
-        }, async (prepared, index)=>{
-            try {
-                projectResults[index] = await runPreparedProject(prepared, index);
-            } catch (error) {
-                recordInfrastructureError(error);
-                throw error;
-            }
+        const states = await runPreparedFiles({
+            projects: preparedProjects,
+            maxConcurrentFiles: effectiveConcurrency,
+            maxConcurrentProjects: definition.test.maxConcurrentProjects ?? 1 / 0,
+            setupTimeoutMs: definition.test.setupTimeout,
+            teardownTimeoutMs,
+            signal: rootController.signal,
+            runDir,
+            preflightFailed: 'run' === plan.preflightScope && totalErrors > 0,
+            shouldBail: bailReached,
+            addFailedCases,
+            announceProject,
+            onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
+            onInfrastructureError: recordInfrastructureError
+        });
+        states.forEach((state, index)=>{
+            projectResults[index] = buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle);
         });
     } catch (error) {
         recordInfrastructureError(error);