@midscene/test 1.15.6 → 1.16.1

package/dist/lib/test-cli.js

@@ -29,7 +29,7 @@ var __webpack_modules__ = {
     },
     "./package.json" (module) {
         module.exports = {
-            rE: "1.15.6"
+            rE: "1.16.1"
         };
     }
 };
@@ -353,9 +353,19 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
         if (!Number.isInteger(value) || value < 0) throw new TypeError(`Midscene config ${label} must be a non-negative integer.`);
         return value;
     };
-    const validateExecutionProjects = (value, defaultSetup, globalNodes)=>{
+    const validateConcurrency = (value, label)=>{
+        if (null === value) return null;
+        return validatePositiveInteger(value, 1, label);
+    };
+    const validateFileConcurrency = (maxConcurrentFiles, fileSetup, label)=>{
+        if (maxConcurrentFiles > 1 && !fileSetup) throw new TypeError(`Midscene config ${label}.fileSetup is required when the effective maxConcurrentFiles is greater than 1.`);
+    };
+    const validateExecutionProjects = (value, defaultSetup, defaultFileSetup, globalNodes, test)=>{
         if (void 0 === value) {
             const setup = validateSetup(defaultSetup, 'setup');
+            const fileSetup = validateSetup(defaultFileSetup, 'fileSetup');
+            const maxConcurrentFiles = test.maxConcurrentFiles ?? 1 / 0;
+            validateFileConcurrency(maxConcurrentFiles, fileSetup, 'default');
             return {
                 hasExplicitProjects: false,
                 projects: Object.freeze([
@@ -365,6 +375,10 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                         ...setup ? {
                             setup
                         } : {},
+                        ...fileSetup ? {
+                            fileSetup
+                        } : {},
+                        maxConcurrentFiles,
                         tags: Object.freeze({
                             include: [],
                             exclude: []
@@ -377,6 +391,7 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             };
         }
         if (void 0 !== defaultSetup) throw new TypeError('Midscene config setup cannot be used together with projects. Move setup to projects[].setup.');
+        if (void 0 !== defaultFileSetup) throw new TypeError('Midscene config fileSetup cannot be used together with projects. Move fileSetup to projects[].fileSetup.');
         if (!Array.isArray(value) || 0 === value.length) throw new TypeError('Midscene config projects must be a non-empty array when declared.');
         const names = new Set();
         const projects = value.map((candidate, index)=>{
@@ -385,6 +400,8 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             rejectUnknownKeys(candidate, [
                 'name',
                 'setup',
+                'fileSetup',
+                'maxConcurrentFiles',
                 'nodes',
                 'files',
                 'tags',
@@ -401,10 +418,18 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                 ...globalNodes.definitions().filter((node)=>!localNodes.has(node.name)),
                 ...localNodes.definitions()
             ]);
+            const fileSetup = validateSetup(candidate.fileSetup, `${label}.fileSetup`);
+            const globalFileLimit = test.maxConcurrentFiles ?? 1 / 0;
+            const maxConcurrentFiles = Math.min(globalFileLimit, validatePositiveInteger(candidate.maxConcurrentFiles, globalFileLimit, `${label}.maxConcurrentFiles`));
+            validateFileConcurrency(maxConcurrentFiles, fileSetup, label);
             return Object.freeze({
                 projectId: projectIdFromIndex(index),
                 name: candidate.name,
                 nodes,
+                ...fileSetup ? {
+                    fileSetup
+                } : {},
+                maxConcurrentFiles,
                 ...files ? {
                     files
                 } : {},
@@ -421,11 +446,25 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             hasExplicitProjects: true
         };
     };
-    const validateTestOptions = (value)=>{
+    const validateTestOptions = (value, hasExplicitProjects)=>{
         if (void 0 !== value && !isRecord(value)) throw new TypeError('Midscene config test must be an object.');
         const candidate = value ?? {};
+        if (void 0 !== candidate.maxConcurrency) {
+            const limit = 'number' == typeof candidate.maxConcurrency && Number.isInteger(candidate.maxConcurrency) && candidate.maxConcurrency > 0 ? candidate.maxConcurrency : 1;
+            throw new TypeError([
+                'Midscene config test.maxConcurrency is no longer supported.',
+                'It controlled project concurrency, not file concurrency. Remove it and use:',
+                `  test: { maxConcurrentProjects: ${limit}, maxConcurrentFiles: ${hasExplicitProjects ? limit : 1} }`,
+                hasExplicitProjects ? '  Set projects[].maxConcurrentFiles to 1 to keep files sequential within each project.' : '  The implicit default project keeps running files sequentially.',
+                hasExplicitProjects ? 'For parallel files, configure fileSetup and raise both test.maxConcurrentFiles and projects[].maxConcurrentFiles.' : 'For parallel files, configure fileSetup and raise test.maxConcurrentFiles.',
+                'Migration guide: https://midscenejs.com/midscene-test/configuration.html#existing-configurations'
+            ].join('\n'));
+        }
         return Object.freeze({
-            maxConcurrency: validatePositiveInteger(candidate.maxConcurrency, 1, 'test.maxConcurrency'),
+            maxConcurrentFiles: validateConcurrency(candidate.maxConcurrentFiles, 'test.maxConcurrentFiles'),
+            maxConcurrentProjects: validateConcurrency(candidate.maxConcurrentProjects, 'test.maxConcurrentProjects'),
+            setupTimeout: validatePositiveInteger(candidate.setupTimeout, 120000, 'test.setupTimeout'),
+            teardownTimeout: validatePositiveInteger(candidate.teardownTimeout, 30000, 'test.teardownTimeout'),
             bail: validateNonNegativeInteger(candidate.bail, 0, 'test.bail'),
             testTimeout: validatePositiveInteger(candidate.testTimeout, 120000, 'test.testTimeout')
         });
@@ -455,16 +494,18 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
         if ('testRunner' in definition) throw new TypeError('Midscene config testRunner is not supported. Rename it to test.');
         rejectUnknownKeys(definition, [
             'setup',
+            'fileSetup',
             'projects',
             'test',
             'output',
             'nodes'
         ], 'root');
         const nodes = new NodeRegistry(definition.nodes);
-        const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, nodes);
+        const test = validateTestOptions(definition.test, void 0 !== definition.projects);
+        const resolvedProjects = validateExecutionProjects(definition.projects, definition.setup, definition.fileSetup, nodes, test);
         return {
             ...resolvedProjects,
-            test: validateTestOptions(definition.test),
+            test,
             output: validateOutput(definition.output),
             nodes,
             resolveNode: (name)=>nodes.get(name)
@@ -1142,7 +1183,24 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                 defaultTimeoutMs,
                 reportEnabled: true,
                 bindings: {
-                    resolveNode: project.nodes.require.bind(project.nodes)
+                    resolveNode: project.nodes.require.bind(project.nodes),
+                    ...project.fileSetup ? {
+                        documentSetup: {
+                            name: project.fileSetup.name,
+                            setup ({ projectContext, document, signal, onTeardown }) {
+                                return project.fileSetup.setup({
+                                    project,
+                                    projectContext,
+                                    file: document,
+                                    env: Object.freeze({
+                                        ...process.env
+                                    }),
+                                    signal,
+                                    onTeardown
+                                });
+                            }
+                        }
+                    } : {}
                 }
             });
         } catch (error) {
@@ -1348,11 +1406,40 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             ...result.beforeAll,
             ...result.afterAll
         ].some((step)=>step.error && (0, test_runner_namespaceObject.isFatalDeviceError)(step.error)) || (result.teardownErrors ?? []).some(test_runner_namespaceObject.isFatalDeviceError) || !!result.executionErrors?.some(test_runner_namespaceObject.isResourceRecoveryError);
+    const isSharedResourceFailure = (error)=>{
+        const seen = new Set();
+        const pending = [
+            error
+        ];
+        while(pending.length && seen.size < 50){
+            const item = pending.pop();
+            if ('object' != typeof item || null === item || seen.has(item)) continue;
+            seen.add(item);
+            const failure = item;
+            if ('PROJECT_RESOURCE_ERROR' === failure.code) return true;
+            if (failure.cause) pending.push(failure.cause);
+            if (Array.isArray(failure.errors)) pending.push(...failure.errors.slice(0, 50));
+        }
+        return false;
+    };
     async function executeDocumentInvocation(options) {
         const { invocation, project, sinks } = options;
+        const isProjectResourceFailure = options.isProjectResourceFailure ?? (()=>false);
         const { document } = invocation;
         const documentRetry = 'document' === invocation.retry.scope;
         const attempts = [];
+        const fileSetup = invocation.bindings.documentSetup;
+        const documentSetup = options.isProjectResourceFailure && fileSetup ? {
+            ...fileSetup,
+            async setup (ctx) {
+                try {
+                    return await fileSetup.setup(ctx);
+                } catch (error) {
+                    if (isProjectResourceFailure(error)) options.markProjectFatal();
+                    throw error;
+                }
+            }
+        } : fileSetup;
         await (0, test_runner_namespaceObject.runDocumentAttempts)({
             retry: documentRetry ? invocation.retry.count : 0,
             signal: options.signal,
@@ -1363,6 +1450,9 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                 plan: invocation.plan,
                 documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
                 ...invocation.bindings,
+                documentSetup,
+                setupTimeoutMs: options.setupTimeoutMs,
+                teardownTimeoutMs: options.teardownTimeoutMs,
                 project,
                 projectContext: options.projectContext,
                 retry: documentRetry ? 0 : invocation.retry.count,
@@ -1374,7 +1464,7 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                         ...run.beforeEach,
                         ...run.steps,
                         ...run.afterEach
-                    ].some((step)=>step.error && (0, test_runner_namespaceObject.isFatalDeviceError)(step.error)) || (run.teardownErrors ?? []).some(test_runner_namespaceObject.isFatalDeviceError),
+                    ].some((step)=>step.error && ((0, test_runner_namespaceObject.isFatalDeviceError)(step.error) || isProjectResourceFailure(step.error))) || (run.teardownErrors ?? []).some((error)=>(0, test_runner_namespaceObject.isFatalDeviceError)(error) || isProjectResourceFailure(error)),
                 onCaseStart: (collectedCase)=>{
                     options.onProgress(`    [case ${collectedCase.caseIndex + 1}/${document.cases.length}] ${collectedCase.definition.name}`);
                 },
@@ -1383,6 +1473,7 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                     options.onProgress(`${indent}→ ${formatStep(info)}`);
                 },
                 onStepResult: async (info, result)=>{
+                    if (isProjectResourceFailure(result.error)) options.markProjectFatal();
                     await invocation.bindings.onStepResult?.(info, result);
                     options.onProgress(formatStepResult(info, result));
                 },
@@ -1391,7 +1482,12 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                     options.onProgress(`    ${'success' === attempt.status ? '✓' : '✗'} attempt ${attempt.attemptIndex + 1}/${documentRetry ? 1 : invocation.retry.count + 1}: ${attempt.name} (${attempt.durationMs} ms)`);
                 },
                 onCaseOutcome: (outcome)=>{
-                    if (caseHasFatalError(outcome)) options.markProjectFatal();
+                    if (outcome.attempts?.some((attempt)=>[
+                            ...attempt.beforeEach,
+                            ...attempt.steps,
+                            ...attempt.afterEach
+                        ].some((step)=>isProjectResourceFailure(step.error)) || attempt.teardownErrors?.some(isProjectResourceFailure))) options.markProjectFatal();
+                    if (!options.isolatedContext && caseHasFatalError(outcome)) options.markProjectFatal();
                     if (!documentRetry && 'failed' === outcome.status) options.addFailedCases(1);
                 },
                 onDocumentResult: options.onDocumentResult
@@ -1400,7 +1496,7 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                     const partial = error.result;
                     const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>(0, test_runner_namespaceObject.isResourceRecoveryError)(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
                     if (resourceOnly) {
-                        if (error.errors.some(test_runner_namespaceObject.isResourceRecoveryError)) options.markProjectFatal();
+                        if (!options.isolatedContext && error.errors.some(test_runner_namespaceObject.isResourceRecoveryError)) options.markProjectFatal();
                         return partial;
                     }
                     if (documentRetry) partial.document.retryDecision = (0, test_runner_namespaceObject.decideRetry)({
@@ -1429,7 +1525,17 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
                 }
                 throw error;
             });
-            if (documentHasFatalError(execution.document)) options.markProjectFatal();
+            if (!options.isolatedContext && documentHasFatalError(execution.document)) options.markProjectFatal();
+            if ([
+                ...execution.document.beforeAll,
+                ...execution.document.afterAll,
+                ...(execution.document.hostErrors ?? []).map(({ error })=>({
+                        error
+                    })),
+                ...(execution.document.teardownErrors ?? []).map((error)=>({
+                        error
+                    }))
+            ].some(({ error })=>isProjectResourceFailure(error))) options.markProjectFatal();
             const owners = [
                 execution.document,
                 ...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
@@ -1437,7 +1543,10 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
             const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
             const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
-            if (cleanupPending || cleanupFailed || operationPending) options.markProjectFatal();
+            if (cleanupPending || operationPending) {
+                options.markProjectFatal();
+                options.onUnresolvedResources?.();
+            } else if (cleanupFailed && !options.isolatedContext) options.markProjectFatal();
             const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
             const decision = (0, test_runner_namespaceObject.decideRetry)({
                 scope: 'document',
@@ -1490,6 +1599,231 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
         if (documentRetry && failedCases) options.addFailedCases(failedCases);
         else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
     }
+    async function runPreparedFiles(options) {
+        const states = options.projects.map((prepared)=>({
+                prepared,
+                cases: [],
+                documents: [],
+                cursor: 0,
+                active: 0,
+                initializing: false,
+                finished: false,
+                fatal: false,
+                setupFailed: false
+            }));
+        let nextProject = 0;
+        let resourcesUnresolved = false;
+        let occupied = 0;
+        let occupiedProjects = 0;
+        const cleanupQueue = [];
+        const release = ()=>{
+            occupied--;
+            while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+                occupied++;
+                cleanupQueue.shift()();
+            }
+            wake?.();
+        };
+        let wake;
+        const running = new Set();
+        const stopped = (state)=>options.signal.aborted || options.shouldBail() || state.fatal;
+        const invalid = (state)=>options.preflightFailed || state.prepared.collectionErrors.length > 0;
+        const unresolved = ()=>{
+            if (resourcesUnresolved) return;
+            resourcesUnresolved = true;
+            options.onInfrastructureError(new test_runner_namespaceObject.WorkflowError('Resources are still running or cleanup is pending. No new test files will start.', {
+                code: 'UNRESOLVED_FILE_RESOURCES'
+            }));
+        };
+        const finishProject = async (state)=>{
+            if (!state.runtime || state.finished) return;
+            state.finished = true;
+            const failed = state.fatal || options.signal.aborted || state.cases.some((item)=>'success' !== item.status) || state.documents.some((item)=>'success' !== item.status);
+            state.lifecycle = await state.runtime.finish(failed ? 'failed' : 'success');
+            const releaseProject = ()=>{
+                occupiedProjects--;
+                wake?.();
+            };
+            if (state.lifecycle.cleanup?.status === 'pending' || state.lifecycle.pendingOperations?.length) {
+                unresolved();
+                Promise.all([
+                    (0, test_runner_namespaceObject.waitForScopeOperations)(state.runtime.signal),
+                    (0, test_runner_namespaceObject.getResourceCleanupCompletion)(state.runtime.signal)
+                ]).then(releaseProject, (error)=>{
+                    options.onInfrastructureError(error);
+                    releaseProject();
+                });
+            } else releaseProject();
+        };
+        const admit = ()=>{
+            for(let offset = 0; offset < states.length; offset++){
+                const index = (nextProject + offset) % states.length;
+                const state = states[index];
+                if (invalid(state) || state.finished || state.initializing) continue;
+                if (!state.runtime && occupiedProjects >= options.maxConcurrentProjects) continue;
+                const cleanupOnly = !!state.runtime && 0 === state.active && (stopped(state) || state.cursor === state.prepared.invocations.length);
+                if (!cleanupOnly && (stopped(state) || state.cursor >= state.prepared.invocations.length || state.active >= (state.prepared.project.maxConcurrentFiles ?? options.maxConcurrentFiles))) continue;
+                nextProject = (index + 1) % states.length;
+                occupied++;
+                state.active++;
+                if (!state.runtime && !cleanupOnly) {
+                    occupiedProjects++;
+                    state.initializing = true;
+                }
+                const invocationIndex = cleanupOnly ? -1 : state.cursor++;
+                return {
+                    state,
+                    index,
+                    invocationIndex
+                };
+            }
+        };
+        const run = async ({ state, index, invocationIndex })=>{
+            const { prepared } = state;
+            const { project } = prepared;
+            let fileResourcesPending = false;
+            try {
+                if (invocationIndex < 0) return void await finishProject(state);
+                if (!state.runtime) {
+                    state.runtime = (0, test_runner_namespaceObject.createProjectRuntime)({
+                        project,
+                        setup: project.setup,
+                        signal: options.signal,
+                        setupTimeoutMs: options.setupTimeoutMs,
+                        teardownTimeoutMs: options.teardownTimeoutMs
+                    });
+                    (0, external_node_events_namespaceObject.setMaxListeners)(Math.max(10, Math.min(prepared.invocations.length, project.maxConcurrentFiles ?? options.maxConcurrentFiles) + 1), state.runtime.signal);
+                    options.announceProject(prepared, index);
+                    await state.runtime.start();
+                    state.initializing = false;
+                    if (!state.runtime.canRun) {
+                        state.fatal = true;
+                        state.setupFailed = true;
+                    }
+                    wake?.();
+                }
+                if (stopped(state)) return;
+                const invocation = prepared.invocations[invocationIndex];
+                options.onProgress(prepared, `  [document ${invocationIndex + 1}/${prepared.invocations.length}] ${invocation.document.sourcePath}`);
+                await executeDocumentInvocation({
+                    invocation,
+                    project,
+                    projectContext: state.runtime.context,
+                    runDir: options.runDir,
+                    signal: state.runtime.signal,
+                    rootSignal: options.signal,
+                    shouldBail: options.shouldBail,
+                    isProjectFatal: ()=>state.fatal,
+                    markProjectFatal: ()=>{
+                        if (state.fatal) return;
+                        state.fatal = true;
+                        state.runtime?.abort(new test_runner_namespaceObject.WorkflowError(`Project "${project.name}" shared execution environment is unavailable.`, {
+                            code: 'PROJECT_RESOURCE_ERROR'
+                        }));
+                    },
+                    addFailedCases: options.addFailedCases,
+                    onProgress: (message)=>options.onProgress(prepared, message),
+                    sinks: state,
+                    isolatedContext: void 0 !== project.fileSetup,
+                    isProjectResourceFailure: isSharedResourceFailure,
+                    setupTimeoutMs: options.setupTimeoutMs,
+                    teardownTimeoutMs: options.teardownTimeoutMs,
+                    onUnresolvedResources: ()=>{
+                        fileResourcesPending = true;
+                        unresolved();
+                    }
+                });
+            } catch (error) {
+                state.fatal = true;
+                fileResourcesPending = !!state.runtime && (0, test_runner_namespaceObject.getPendingScopeOperations)(state.runtime.signal).length > 0;
+                options.onInfrastructureError(error);
+            } finally{
+                state.initializing = false;
+                try {
+                    if (1 === state.active && (stopped(state) || state.cursor === prepared.invocations.length)) await finishProject(state);
+                } catch (error) {
+                    options.onInfrastructureError(error);
+                }
+                state.active--;
+                if (state.runtime && (fileResourcesPending || state.lifecycle?.cleanup?.status === 'pending')) (0, test_runner_namespaceObject.waitForScopeOperations)(state.runtime.signal).then(release, (error)=>{
+                    options.onInfrastructureError(error);
+                    release();
+                });
+                else release();
+                wake?.();
+            }
+        };
+        while(true){
+            const changed = new Promise((resolve)=>{
+                wake = resolve;
+            });
+            while(occupied < options.maxConcurrentFiles){
+                const job = admit();
+                if (!job) break;
+                const task = run(job);
+                running.add(task);
+                const settled = ()=>{
+                    running.delete(task);
+                    wake?.();
+                };
+                task.then(settled, (error)=>{
+                    options.onInfrastructureError(error);
+                    settled();
+                });
+            }
+            if (!running.size) break;
+            await changed;
+        }
+        wake = void 0;
+        await Promise.all(states.map(async (state)=>{
+            if (!state.runtime || state.finished) return;
+            const admission = new Promise((resolve)=>{
+                cleanupQueue.push(resolve);
+            });
+            (0, test_runner_namespaceObject.trackResourceOperation)({}, admission, state.runtime.signal);
+            while(occupied < options.maxConcurrentFiles && cleanupQueue.length){
+                occupied++;
+                cleanupQueue.shift()();
+            }
+            try {
+                await finishProject(state);
+            } catch (error) {
+                options.onInfrastructureError(error);
+            } finally{
+                Promise.all([
+                    admission,
+                    (0, test_runner_namespaceObject.getResourceCleanupCompletion)(state.runtime.signal)
+                ]).then(release, (error)=>{
+                    options.onInfrastructureError(error);
+                    release();
+                });
+            }
+        }));
+        for (const state of states){
+            const { prepared } = state;
+            if (!state.runtime) options.announceProject(prepared, states.indexOf(state));
+            const completed = new Set(state.cases.map((item)=>item.caseId));
+            const reason = invalid(state) ? 'project-preflight-failed' : state.setupFailed ? 'project-setup-failed' : options.signal.aborted ? 'interrupted' : state.fatal ? 'fatal-error' : 'bail';
+            for (const document of prepared.documents)for (const item of document.cases)if (!completed.has(item.caseId)) state.cases.push({
+                documentId: document.documentId,
+                caseId: item.caseId,
+                projectName: prepared.project.name,
+                name: item.definition.name,
+                sourcePath: item.sourcePath,
+                caseIndex: item.caseIndex,
+                status: 'not-run',
+                notRunReason: reason
+            });
+            const order = new Map(prepared.documents.map((document, i)=>[
+                    document.documentId,
+                    i
+                ]));
+            const position = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
+            state.cases.sort((a, b)=>position(a.documentId) - position(b.documentId) || a.caseIndex - b.caseIndex);
+            state.documents.sort((a, b)=>position(a.documentId) - position(b.documentId));
+        }
+        return states;
+    }
     function createOperationJournal(runDir) {
         const path = (0, external_node_path_.join)(runDir, 'operations.jsonl');
         (0, external_node_fs_.writeFileSync)(path, '', {
@@ -1559,7 +1893,7 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
         const totalCases = preparedProjects.reduce((total, prepared)=>total + prepared.selectedCaseCount, 0);
         const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
         progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
-        const effectiveConcurrency = Math.min(definition.test.maxConcurrency, preparedProjects.length);
+        const effectiveConcurrency = Math.min(definition.test.maxConcurrentFiles ?? 1 / 0, Math.max(1, totalDocuments));
         await writeTestRunPlan(runDir, preparedProjects);
         const journal = createOperationJournal(runDir);
         const rootController = new AbortController();
@@ -1573,6 +1907,18 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
         process.on('SIGTERM', sigterm);
         let failedCaseCount = 0;
         const bailReached = ()=>definition.test.bail > 0 && failedCaseCount >= definition.test.bail;
+        let shutdownDeadline;
+        const beginShutdown = ()=>{
+            shutdownDeadline ??= Date.now() + definition.test.teardownTimeout;
+        };
+        rootController.signal.addEventListener('abort', beginShutdown, {
+            once: true
+        });
+        const addFailedCases = (count)=>{
+            failedCaseCount += count;
+            if (bailReached()) beginShutdown();
+        };
+        const teardownTimeoutMs = ()=>Math.max(0, Math.min(definition.test.teardownTimeout, void 0 === shutdownDeadline ? 1 / 0 : shutdownDeadline - Date.now()));
         let hasInfrastructureError = false;
         const infrastructureErrors = [];
         const recordInfrastructureError = (error)=>{
@@ -1634,112 +1980,25 @@ Each Node's returned \`data\` is saved on that Step's result. Results are not na
             announceProject(prepared, projectIndex);
             return buildProjectResult(prepared, notRunSuite(prepared, reason), []);
         };
-        const runPreparedProject = async (prepared, projectIndex)=>{
-            const { project } = prepared;
-            announceProject(prepared, projectIndex);
-            const projectProgress = (message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${project.name}]${message}` : message));
-            const cases = [];
-            const documents = [];
-            let lifecycle;
-            let projectFatal = false;
-            if (prepared.collectionErrors.length > 0 || 'run' === plan.preflightScope && totalErrors > 0) cases.push(...notRunSuite(prepared, 'project-preflight-failed'));
-            else if (rootController.signal.aborted) cases.push(...notRunSuite(prepared, 'interrupted'));
-            else if (bailReached()) cases.push(...notRunSuite(prepared, 'bail'));
-            else {
-                const runtime = (0, test_runner_namespaceObject.createProjectRuntime)({
-                    project,
-                    setup: project.setup,
-                    signal: rootController.signal
-                });
-                let hasProjectExecutionError = false;
-                try {
-                    await runtime.start();
-                    if (runtime.canRun) {
-                        const runDocument = async (invocation, documentIndex)=>{
-                            const { document } = invocation;
-                            if (rootController.signal.aborted || bailReached() || projectFatal) {
-                                const reason = rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                                cases.push(...document.cases.map((item)=>asNotRun(document.documentId, item, project.name, reason)));
-                                return;
-                            }
-                            projectProgress(`  [document ${documentIndex + 1}/${prepared.documents.length}] ${document.sourcePath}`);
-                            await executeDocumentInvocation({
-                                invocation,
-                                project,
-                                projectContext: runtime.context,
-                                runDir,
-                                signal: runtime.signal,
-                                rootSignal: rootController.signal,
-                                shouldBail: bailReached,
-                                isProjectFatal: ()=>projectFatal,
-                                markProjectFatal: ()=>{
-                                    projectFatal = true;
-                                },
-                                addFailedCases: (count)=>{
-                                    failedCaseCount += count;
-                                },
-                                onProgress: projectProgress,
-                                sinks: {
-                                    cases,
-                                    documents
-                                }
-                            });
-                        };
-                        let prerequisiteFailed = false;
-                        const prerequisite = prepared.invocations.find((invocation)=>invocation.document.documentId === prepared.prerequisiteDocumentId);
-                        const prerequisiteCount = prerequisite ? 1 : 0;
-                        if (prerequisite) {
-                            await runDocument(prerequisite, 0);
-                            prerequisiteFailed = documents.at(-1)?.status === 'failed' || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status);
-                        }
-                        if (!prerequisiteFailed) await (0, test_runner_namespaceObject.runConcurrentJobs)(prepared.invocations.filter((invocation)=>invocation !== prerequisite), {
-                            concurrency: prepared.documentConcurrency,
-                            shouldStop: ()=>rootController.signal.aborted || bailReached() || projectFatal
-                        }, async (invocation, index)=>{
-                            try {
-                                await runDocument(invocation, index + prerequisiteCount);
-                            } catch (error) {
-                                recordInfrastructureError(error);
-                                throw error;
-                            }
-                        });
-                        const completed = new Set(cases.map((item)=>item.caseId));
-                        const reason = prerequisiteFailed ? 'project-setup-failed' : rootController.signal.aborted ? 'interrupted' : projectFatal ? 'fatal-error' : 'bail';
-                        cases.push(...notRunSuite(prepared, reason).filter((item)=>!completed.has(item.caseId)));
-                    } else cases.push(...notRunSuite(prepared, rootController.signal.aborted ? 'interrupted' : 'project-setup-failed'));
-                } catch (error) {
-                    hasProjectExecutionError = true;
-                    recordInfrastructureError(error);
-                } finally{
-                    const hasFailure = hasProjectExecutionError || latestById(cases, (item)=>item.caseId).some((item)=>'success' !== item.status) || latestById(documents, (item)=>item.documentId).some((item)=>'failed' === item.status) || projectFatal || rootController.signal.aborted;
-                    lifecycle = await runtime.finish(hasFailure ? 'failed' : 'success');
-                }
-                if (hasProjectExecutionError) {
-                    const completed = new Set(cases.map((item)=>item.caseId));
-                    cases.push(...notRunSuite(prepared, 'interrupted').filter((item)=>!completed.has(item.caseId)));
-                }
-            }
-            const order = new Map(prepared.documents.map((document, index)=>[
-                    document.documentId,
-                    index
-                ]));
-            const documentOrder = (id)=>order.get(id) ?? Number.MAX_SAFE_INTEGER;
-            cases.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || a.caseIndex - b.caseIndex);
-            documents.sort((a, b)=>documentOrder(a.documentId) - documentOrder(b.documentId) || (a.attemptIndex ?? 0) - (b.attemptIndex ?? 0));
-            return buildProjectResult(prepared, cases, documents, lifecycle);
-        };
         const projectResults = new Array(preparedProjects.length);
         try {
-            await (0, test_runner_namespaceObject.runConcurrentJobs)(preparedProjects, {
-                concurrency: effectiveConcurrency,
-                shouldStop: ()=>hasInfrastructureError || rootController.signal.aborted || bailReached()
-            }, async (prepared, index)=>{
-                try {
-                    projectResults[index] = await runPreparedProject(prepared, index);
-                } catch (error) {
-                    recordInfrastructureError(error);
-                    throw error;
-                }
+            const states = await runPreparedFiles({
+                projects: preparedProjects,
+                maxConcurrentFiles: effectiveConcurrency,
+                maxConcurrentProjects: definition.test.maxConcurrentProjects ?? 1 / 0,
+                setupTimeoutMs: definition.test.setupTimeout,
+                teardownTimeoutMs,
+                signal: rootController.signal,
+                runDir,
+                preflightFailed: 'run' === plan.preflightScope && totalErrors > 0,
+                shouldBail: bailReached,
+                addFailedCases,
+                announceProject,
+                onProgress: (prepared, message)=>runInfrastructureCallback(()=>progress(effectiveConcurrency > 1 ? `[${prepared.project.name}]${message}` : message)),
+                onInfrastructureError: recordInfrastructureError
+            });
+            states.forEach((state, index)=>{
+                projectResults[index] = buildProjectResult(state.prepared, state.cases, state.documents, state.lifecycle);
             });
         } catch (error) {
             recordInfrastructureError(error);