@midscene/test 1.15.2-beta-20261012123219.0 → 1.15.2

package/dist/es/cli/index.mjs

@@ -1,11 +1,11 @@
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import { require as api_require } from "tsx/cjs/api";
-import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, isFatalDeviceError, normalizeSteps, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument } from "@midscene/core/internal/test-runner";
-import { existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { NodeRegistry, WorkflowError, WorkflowExecutionFailure, WorkflowParseError, WorkflowPublicationError, asExecutionError, buildTestRunReportDump, buildTestRunReportPlan, classifyWorkflowDocumentFormat, collectTestRunReportSources, createProjectRuntime, decideRetry, isFatalDeviceError, isReportSecretKey, isResourceRecoveryError, normalizeSteps, observeResourceOperations, runConcurrentJobs, runDocumentAttempts, runWorkflowDocument, sanitizeReportError, sanitizeReportText } from "@midscene/core/internal/test-runner";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { globSync } from "tinyglobby";
-import { JSON_SCHEMA, load } from "js-yaml";
 import { createHash, randomUUID } from "node:crypto";
+import { JSON_SCHEMA, load } from "js-yaml";
 import { mkdir, writeFile } from "node:fs/promises";
 import { setMaxListeners } from "node:events";
 import { TestRunReportAssembler, calculateTestRunHealth } from "@midscene/core/report";
@@ -304,6 +304,69 @@ const createDocumentInvocationId = (projectId, sourcePath, invocationIndex = 0)=
         ] : []
     ])).digest('hex');
 const normalize_normalizeSteps = normalizeSteps;
+function parseWorkflowSource(source) {
+    const stack = [];
+    const nodes = [];
+    const sensitiveNodes = new Set();
+    const sensitiveValues = new Set();
+    const redactedPaths = new Set();
+    const sourceOffset = 0xfeff === source.charCodeAt(0) ? 1 : 0;
+    const parsed = load(source, {
+        schema: JSON_SCHEMA,
+        listener (event, state) {
+            if ('open' === event) return void stack.push({
+                start: state.position + sourceOffset,
+                end: 0,
+                value: null,
+                children: []
+            });
+            const node = stack.pop();
+            node.end = state.position + sourceOffset;
+            node.value = state.result;
+            stack.at(-1)?.children.push(node);
+            nodes.push(node);
+            if ('mapping' !== state.kind) return;
+            for(let index = 0; index < node.children.length; index += 2){
+                const key = node.children[index].value;
+                const value = node.children[index + 1];
+                if ('string' == typeof key && isReportSecretKey(key) && value) {
+                    sensitiveNodes.add(value);
+                    if (null !== value.value) sensitiveValues.add(value.value);
+                    redactedPaths.add(`$..[${JSON.stringify(key)}]`);
+                }
+            }
+        }
+    });
+    const ranges = nodes.filter((node)=>sensitiveNodes.has(node) || sensitiveValues.has(node.value)).sort((left, right)=>left.start - right.start || right.end - left.end);
+    let offset = 0;
+    let redacted = '';
+    for (const range of ranges){
+        if (range.start < offset) continue;
+        const text = source.slice(range.start, range.end);
+        const leading = text.match(/^\s*/)?.[0] ?? '';
+        const trailing = text.slice(leading.length).match(/\s*$/)?.[0] ?? '';
+        redacted += `${source.slice(offset, range.start)}${leading}"[REDACTED]"${trailing}`;
+        offset = range.end;
+    }
+    redacted += source.slice(offset);
+    const value = sanitizeReportText(redacted, 65536);
+    return {
+        parsed,
+        sourceYaml: {
+            value,
+            ...redactedPaths.size ? {
+                redactedPaths: [
+                    ...redactedPaths
+                ]
+            } : {},
+            ...value !== redacted ? {
+                truncatedPaths: [
+                    '$'
+                ]
+            } : {}
+        }
+    };
+}
 const interpolationExpression = /\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}|\$\{([^{}]+)\}/g;
 const exactEnvExpression = /^\$\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}$/;
 const exactProjectExpression = /^\$\{([^{}]+)\}$/;
@@ -378,16 +441,15 @@ const collect_rejectUnknownKeys = (value, allowed, label)=>{
 const createCaseId = (projectId, sourcePath, caseIndex)=>createCaseInvocationId(projectId, sourcePath, caseIndex);
 const createWorkflowDocumentId = (projectId, sourcePath)=>createDocumentInvocationId(projectId, sourcePath);
 function collectWorkflowDocument(source, options) {
-    let parsed;
+    let snapshot;
     try {
-        parsed = load(readFileSync(source.absolutePath, 'utf8'), {
-            schema: JSON_SCHEMA
-        });
+        snapshot = parseWorkflowSource(readFileSync(source.absolutePath, 'utf8'));
     } catch (error) {
         throw new WorkflowParseError(`Failed to parse workflow document "${source.sourcePath}".`, {
             sourcePath: source.sourcePath
         }, error);
     }
+    const { parsed, sourceYaml } = snapshot;
     if (!isMapping(parsed)) throw new WorkflowParseError('Workflow document must be a mapping.');
     const format = classifyWorkflowDocumentFormat(parsed);
     if ('legacy' === format || 'legacy-batch' === format || 'mixed' === format) throw new NativeWorkflowFormatError(format, source.sourcePath);
@@ -402,6 +464,7 @@ function collectWorkflowDocument(source, options) {
     const sourcePath = source.sourcePath.split(sep).join('/');
     const resolveStepVariables = (step, phase, stepIndex, caseIndex)=>({
             ...step,
+            templateInput: step.input,
             input: resolveWorkflowVariables(step.input, {
                 variables: options.variables,
                 env: options.env ?? process.env,
@@ -487,6 +550,7 @@ function collectWorkflowDocument(source, options) {
         documentId: createWorkflowDocumentId(source.projectId, sourcePath),
         projectId: source.projectId,
         sourcePath,
+        sourceYaml,
         lifecycle,
         cases
     };
@@ -501,6 +565,15 @@ const writeJson = async (path, value)=>{
         throw new WorkflowPublicationError('write-result', path, error);
     }
 };
+const writeTestRunPlan = (runDir, projects)=>writeJson(join(runDir, 'plan.json'), {
+        schemaVersion: 1,
+        createdAt: new Date().toISOString(),
+        projects: projects.map((item)=>({
+                projectId: item.project.projectId,
+                name: item.project.name,
+                documents: item.plans ?? []
+            }))
+    });
 const toPosix = (value)=>value.split(sep).join('/');
 const caseAttemptResultPath = (projectId, documentId, result)=>toPosix(join(projectId, 'documents', documentId, 'cases', result.caseId, `${result.runId}.json`));
 const workflowDocumentResultPath = (result)=>toPosix(join(result.projectId, 'documents', result.documentId, void 0 === result.attemptIndex ? 'document.json' : `${result.documentRunId}.json`));
@@ -530,7 +603,7 @@ const relativeToSummary = (summaryPath, target)=>{
     const value = relative(dirname(summaryPath), target);
     return toPosix(value || '.');
 };
-const errorJson = (error)=>error instanceof Error && 'toJSON' in error ? error.toJSON() : error;
+const errorJson = sanitizeReportError;
 const writeTestProjectRunResult = (options)=>{
     const { result } = options;
     const fact = (path)=>toPosix(path);
@@ -550,13 +623,17 @@ const writeTestProjectRunResult = (options)=>{
             configPath: options.configPath
         } : {},
         factsRoot: '.',
+        plan: 'plan.json',
+        ...result.operationLog ? {
+            operationLog: {
+                ...result.operationLog,
+                path: relativeToSummary(result.summaryPath, result.operationLog.path)
+            }
+        } : {},
         reportDir: relativeToSummary(result.summaryPath, result.reportDir),
         ...result.reportPath ? {
             report: relativeToSummary(result.summaryPath, result.reportPath)
         } : {},
-        ...result.markdownPath ? {
-            markdown: relativeToSummary(result.summaryPath, result.markdownPath)
-        } : {},
         summary: result.summary,
         projects: result.projects.map((project)=>({
                 projectId: project.projectId,
@@ -574,6 +651,12 @@ const writeTestProjectRunResult = (options)=>{
                         startedAt: project.lifecycle.startedAt,
                         endedAt: project.lifecycle.endedAt,
                         durationMs: project.lifecycle.durationMs,
+                        ...project.lifecycle.cleanup ? {
+                            cleanup: project.lifecycle.cleanup
+                        } : {},
+                        ...project.lifecycle.pendingOperations ? {
+                            pendingOperations: project.lifecycle.pendingOperations
+                        } : {},
                         ...project.lifecycle.setupError ? {
                             setupError: errorJson(project.lifecycle.setupError)
                         } : {},
@@ -600,6 +683,18 @@ const writeTestProjectRunResult = (options)=>{
                                     attemptId: attempt.runId,
                                     attemptIndex: attempt.attemptIndex,
                                     status: attempt.status,
+                                    ...attempt.cleanup ? {
+                                        cleanup: attempt.cleanup
+                                    } : {},
+                                    ...attempt.pendingOperations ? {
+                                        pendingOperations: attempt.pendingOperations
+                                    } : {},
+                                    ...attempt.executionStatus ? {
+                                        executionStatus: attempt.executionStatus
+                                    } : {},
+                                    ...attempt.retryDecision ? {
+                                        retryDecision: attempt.retryDecision
+                                    } : {},
                                     resultFile: fact(caseAttemptResultPath(project.projectId, outcome.documentId, attempt)),
                                     ...attempt.reportPaths?.length ? {
                                         reports: attempt.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
@@ -615,6 +710,18 @@ const writeTestProjectRunResult = (options)=>{
                         },
                         sourcePath: document.sourcePath,
                         status: document.status,
+                        ...document.cleanup ? {
+                            cleanup: document.cleanup
+                        } : {},
+                        ...document.pendingOperations ? {
+                            pendingOperations: document.pendingOperations
+                        } : {},
+                        ...document.executionStatus ? {
+                            executionStatus: document.executionStatus
+                        } : {},
+                        ...document.retryDecision ? {
+                            retryDecision: document.retryDecision
+                        } : {},
                         resultFile: fact(workflowDocumentResultPath(document)),
                         ...document.reportPaths?.length ? {
                             reports: document.reportPaths.map((path)=>relativeToSummary(result.summaryPath, path))
@@ -754,6 +861,7 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
     });
     const collectionErrors = [];
     const invocations = [];
+    const plans = [];
     let filteredCaseCount = 0;
     if (0 === sources.length) {
         const error = asCollectionError(project.projectId, project.name, '<project>', new WorkflowParseError(`No workflow YAML files found for project "${project.name}" in ${projectRoot}.`, {
@@ -767,7 +875,17 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
         const collected = await options.collectDocument?.(source, project);
         if (collected) {
             filteredCaseCount += collected.filteredCaseCount;
-            if (collected.invocation) invocations.push(collected.invocation);
+            if (collected.invocation) {
+                const plan = buildTestRunReportPlan(collected.invocation.document, {
+                    defaultTimeoutMs: collected.invocation.defaultTimeoutMs,
+                    retry: collected.invocation.retry
+                });
+                invocations.push({
+                    ...collected.invocation,
+                    plan
+                });
+                plans.push(plan);
+            }
             continue;
         }
         const document = collectWorkflowDocument(source, {
@@ -779,8 +897,18 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
             document,
             filtered: 0
         } : filterDocumentCases(document, project);
+        const plan = buildTestRunReportPlan(document, {
+            selectedCaseIds: filtered.document?.cases.map((item)=>item.caseId) ?? [],
+            defaultTimeoutMs,
+            retry: {
+                scope: 'case',
+                count: project.retry
+            }
+        });
+        plans.push(plan);
         filteredCaseCount += filtered.filtered;
         if (filtered.document) invocations.push({
+            plan,
             document: filtered.document,
             retry: {
                 scope: 'case',
@@ -813,6 +941,7 @@ async function prepareProject(project, projectRoot, runDir, defaultTimeoutMs, op
         sources,
         documents,
         invocations,
+        plans,
         collectionErrors,
         selectedCaseCount: documents.reduce((total, document)=>total + document.cases.length, 0),
         filteredCaseCount
@@ -893,103 +1022,6 @@ async function prepareTestRun(options) {
         onProgress: options.onProgress ?? (()=>{})
     };
 }
-const result_markdown_text = (value)=>value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;').replaceAll('|', '&#124;').replaceAll('[', '&#91;').replaceAll(']', '&#93;').replaceAll('*', '&#42;').replaceAll('_', '&#95;').replaceAll('`', '&#96;').replaceAll('\\', '&#92;').replace(/[\r\n]+/g, ' ');
-const duration = (ms)=>{
-    const seconds = Math.round(ms / 1000);
-    return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
-};
-const href = (from, target)=>relative(dirname(from), target).split(sep).map(encodeURIComponent).join('/');
-function renderTestResultMarkdown(dump, options = {}) {
-    const { summary } = dump;
-    const outcome = 'success' === dump.status ? `✅ **${summary.passed}/${summary.total} passed**` : `❌ **${summary.failed} failed** · ${summary.passed} passed · ${summary.notRun} not run`;
-    const lines = [
-        '## Midscene Test',
-        '',
-        `${outcome} · ${duration(dump.durationMs)}`,
-        ''
-    ];
-    if (summary.filtered) lines.push(`**Filtered:** ${summary.filtered}`, '');
-    const infrastructure = [
-        summary.collectionErrors ? `${summary.collectionErrors} collection errors` : '',
-        summary.documentFailures ? `${summary.documentFailures} document failures` : '',
-        summary.projectFailures ? `${summary.projectFailures} project failures` : ''
-    ].filter(Boolean);
-    if (infrastructure.length) lines.push(`**Infrastructure:** ${infrastructure.join(' · ')}`, '');
-    const groups = [];
-    const errors = [];
-    const recordSteps = (label, steps)=>{
-        for (const step of steps)if ('failed' === step.status) errors.push(`${label} / ${step.node}: ${step.error?.message ?? 'Step failed'}`);
-    };
-    for (const project of dump.projects){
-        if (project.lifecycle?.setupError) errors.push(`${project.name}: ${project.lifecycle.setupError.message}`);
-        for (const error of project.lifecycle?.teardownErrors ?? [])errors.push(`${project.name}: ${error.message}`);
-        for (const error of project.collectionErrors)errors.push(`${project.name} / ${error.sourcePath}: ${error.error.message}`);
-        for (const document of project.documents){
-            const name = document.sourcePath.replace(/\\/g, '/').replace(/\.(?:yaml|yml)$/, '');
-            const group = {
-                name: dump.projects.length > 1 ? `${project.name} / ${name}` : name,
-                rows: []
-            };
-            groups.push(group);
-            recordSteps(`${project.name} / ${document.sourcePath}`, [
-                ...document.beforeAll,
-                ...document.afterAll
-            ]);
-            for (const error of document.teardownErrors ?? [])errors.push(`${document.sourcePath}: ${error.message}`);
-            for (const failure of document.hostErrors ?? [])errors.push(`${document.sourcePath} / ${failure.phase}: ${failure.error.message}`);
-            for (const testCase of document.cases){
-                const attempt = testCase.attempts.at(-1);
-                const steps = attempt ? [
-                    ...attempt.beforeEach,
-                    ...attempt.steps,
-                    ...attempt.afterEach
-                ] : [];
-                const failedSteps = steps.filter((step)=>'failed' === step.status);
-                const step = failedSteps.at(-1) ?? [
-                    ...steps
-                ].reverse().find((item)=>item.agentDetails?.length) ?? steps.at(-1);
-                const route = new URLSearchParams({
-                    'runner-page': 'case',
-                    'runner-project': project.projectId,
-                    'runner-case': testCase.caseId,
-                    ...step ? {
-                        'runner-step': step.id
-                    } : {}
-                });
-                const target = options.reportHref ? `${options.reportHref}#${route}` : void 0;
-                const name = target ? `[${result_markdown_text(testCase.name)}](<${target}>)` : result_markdown_text(testCase.name);
-                group.rows.push(`  - ${'success' === testCase.status ? '✅' : 'not-run' === testCase.status ? '⏭️' : '❌'} ${name}`);
-            }
-        }
-    }
-    for (const diagnostic of dump.diagnostics ?? [])if ('error' === diagnostic.level) errors.push(diagnostic.message);
-    if (errors.length) lines.push('### Execution errors', '', ...errors.map((error)=>`- ${result_markdown_text(error)}`), '');
-    for (const group of groups)if (group.rows.length) lines.push(`- ${result_markdown_text(group.name)}`, ...group.rows);
-    if (!groups.some((group)=>group.rows.length)) lines.push('No cases were reported.');
-    if (options.reportHref) lines.push('', `[Open the Midscene Test report ↗](<${options.reportHref}>)`);
-    return `${lines.join('\n')}\n`;
-}
-const emptyIndex = ()=>({
-        sources: [],
-        metrics: {
-            modelCallCount: 0,
-            modelTimeMs: 0,
-            promptTokens: 0,
-            cachedInputTokens: 0,
-            completionTokens: 0,
-            totalTokens: 0
-        }
-    });
-async function writeTestResultMarkdown(result, index = emptyIndex()) {
-    const output = join(dirname(result.summaryPath), 'result.md');
-    const dump = buildTestRunReportDump(result, index);
-    await writeFile(output, renderTestResultMarkdown(dump, {
-        ...result.reportPath ? {
-            reportHref: href(output, result.reportPath)
-        } : {}
-    }));
-    return output;
-}
 const stepPosition = (info)=>'case' === info.scope ? info.case : info.document;
 const formatStep = (info)=>{
     const position = stepPosition(info);
@@ -998,6 +1030,7 @@ const formatStep = (info)=>{
 };
 const formatStepResult = (info, result)=>{
     const indent = 'case' === info.scope ? '      ' : '    ';
+    if ('not-run' === result.status) return `${indent}– ${formatStep(info)} (not run: ${result.notRunReason})`;
     const symbol = 'success' === result.status ? '✓' : '✗';
     const error = result.error ? ` — ${result.error.message}` : '';
     const continuation = result.continuedAfterError ? '; continuing' : '';
@@ -1007,11 +1040,11 @@ const caseHasFatalError = (outcome)=>(outcome.attempts ?? []).some((attempt)=>[
             ...attempt.beforeEach,
             ...attempt.steps,
             ...attempt.afterEach
-        ].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError));
+        ].some((step)=>step.error && isFatalDeviceError(step.error)) || (attempt.teardownErrors ?? []).some(isFatalDeviceError) || !!attempt.executionErrors?.some(isResourceRecoveryError));
 const documentHasFatalError = (result)=>[
         ...result.beforeAll,
         ...result.afterAll
-    ].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError);
+    ].some((step)=>step.error && isFatalDeviceError(step.error)) || (result.teardownErrors ?? []).some(isFatalDeviceError) || !!result.executionErrors?.some(isResourceRecoveryError);
 async function executeDocumentInvocation(options) {
     const { invocation, project, sinks } = options;
     const { document } = invocation;
@@ -1024,6 +1057,7 @@ async function executeDocumentInvocation(options) {
     }, async (documentAttemptIndex)=>{
         if (documentRetry) options.onProgress(`    file attempt ${documentAttemptIndex + 1}/${invocation.retry.count + 1}: ${document.sourcePath}`);
         const execution = await runWorkflowDocument(document, {
+            plan: invocation.plan,
             documentAttemptIndex: documentRetry ? documentAttemptIndex : void 0,
             ...invocation.bindings,
             project,
@@ -1061,6 +1095,18 @@ async function executeDocumentInvocation(options) {
         }).catch(async (error)=>{
             if (error instanceof WorkflowExecutionFailure) {
                 const partial = error.result;
+                const resourceOnly = error.errors.length > 0 && error.errors.every((failure)=>isResourceRecoveryError(failure) || partial.document.hostErrors?.some((host)=>'cleanup' === host.phase && (host.error === failure || host.error instanceof Error && host.error.cause === failure)));
+                if (resourceOnly) {
+                    if (error.errors.some(isResourceRecoveryError)) options.markProjectFatal();
+                    return partial;
+                }
+                if (documentRetry) partial.document.retryDecision = decideRetry({
+                    scope: 'document',
+                    retryCount: invocation.retry.count,
+                    attemptIndex: documentAttemptIndex,
+                    status: 'failed',
+                    infrastructureError: true
+                });
                 attempts.push(partial);
                 sinks.documents.push(partial.document);
                 sinks.cases.push(...partial.cases.map((outcome)=>({
@@ -1069,6 +1115,7 @@ async function executeDocumentInvocation(options) {
                         documentRunId: partial.document.documentRunId
                     })));
                 try {
+                    for (const outcome of partial.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
                     await writeWorkflowDocumentResult(options.runDir, partial.document);
                 } catch (publicationError) {
                     throw new WorkflowExecutionFailure(partial, [
@@ -1079,6 +1126,29 @@ async function executeDocumentInvocation(options) {
             }
             throw error;
         });
+        if (documentHasFatalError(execution.document)) options.markProjectFatal();
+        const owners = [
+            execution.document,
+            ...execution.cases.flatMap((outcome)=>outcome.attempts ?? [])
+        ];
+        const cleanupPending = owners.some((owner)=>owner.cleanup?.status === 'pending');
+        const operationPending = owners.some((owner)=>!!owner.pendingOperations?.length);
+        const cleanupFailed = owners.some((owner)=>owner.cleanup?.status === 'failed' || !!owner.teardownErrors?.length || 'hostErrors' in owner && owner.hostErrors?.some(({ phase })=>'cleanup' === phase));
+        if (cleanupPending || cleanupFailed || operationPending) options.markProjectFatal();
+        const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
+        const decision = decideRetry({
+            scope: 'document',
+            retryCount: invocation.retry.count,
+            attemptIndex: documentAttemptIndex,
+            status: failed ? 'failed' : 'success',
+            cleanupPending,
+            cleanupFailed,
+            operationPending,
+            fatalError: options.isProjectFatal(),
+            interrupted: options.signal.aborted || options.rootSignal.aborted,
+            shouldStop: options.shouldBail()
+        });
+        if (documentRetry) execution.document.retryDecision = decision;
         attempts.push(execution);
         sinks.cases.push(...execution.cases.map((outcome)=>({
                 ...outcome,
@@ -1086,15 +1156,30 @@ async function executeDocumentInvocation(options) {
                 documentRunId: execution.document.documentRunId
             })));
         sinks.documents.push(execution.document);
-        await writeWorkflowDocumentResult(options.runDir, execution.document);
-        if (documentHasFatalError(execution.document)) options.markProjectFatal();
-        const cleanupErrors = [
-            ...execution.document.teardownErrors ?? [],
-            ...execution.cases.flatMap((outcome)=>(outcome.attempts ?? []).flatMap((attempt)=>attempt.teardownErrors ?? []))
-        ];
-        if (cleanupErrors.length) throw new WorkflowExecutionFailure(execution, cleanupErrors);
-        const failed = 'failed' === execution.document.status || execution.cases.some((item)=>'success' !== item.status);
-        return failed ? 'failed' : 'success';
+        try {
+            for (const outcome of execution.cases)for (const attempt of outcome.attempts ?? [])await writeCaseAttemptResult(options.runDir, project.projectId, document.documentId, attempt);
+            await writeWorkflowDocumentResult(options.runDir, execution.document);
+        } catch (error) {
+            execution.document.status = 'failed';
+            execution.document.hostErrors = [
+                ...execution.document.hostErrors ?? [],
+                {
+                    phase: 'publication',
+                    error
+                }
+            ];
+            if (documentRetry) execution.document.retryDecision = decideRetry({
+                scope: 'document',
+                retryCount: invocation.retry.count,
+                attemptIndex: documentAttemptIndex,
+                status: 'failed',
+                infrastructureError: true
+            });
+            throw new WorkflowExecutionFailure(execution, [
+                error
+            ]);
+        }
+        return 'retry' === decision.decision ? 'failed' : void 0;
     });
     const final = attempts.at(-1);
     if (!final) return;
@@ -1102,6 +1187,18 @@ async function executeDocumentInvocation(options) {
     if (documentRetry && failedCases) options.addFailedCases(failedCases);
     else if ('failed' === final.document.status && 0 === failedCases) options.addFailedCases(Math.max(1, failedCases));
 }
+function createOperationJournal(runDir) {
+    const path = join(runDir, 'operations.jsonl');
+    writeFileSync(path, '', {
+        flag: 'wx'
+    });
+    return {
+        path,
+        append (event) {
+            appendFileSync(path, `${JSON.stringify(event)}\n`);
+        }
+    };
+}
 const asNotRun = (documentId, collectedCase, projectName, reason)=>({
         documentId,
         caseId: collectedCase.caseId,
@@ -1160,6 +1257,8 @@ async function runPreparedTestProject(plan) {
     const totalErrors = preparedProjects.reduce((total, prepared)=>total + prepared.collectionErrors.length, 0);
     progress(`midscene-test: preflighted ${preparedProjects.length} projects, ${totalDocuments} documents, ${totalCases} cases, ${totalErrors} collection errors`);
     const effectiveConcurrency = Math.min(definition.test.maxConcurrency, preparedProjects.length);
+    await writeTestRunPlan(runDir, preparedProjects);
+    const journal = createOperationJournal(runDir);
     const rootController = new AbortController();
     setMaxListeners(Math.max(10, effectiveConcurrency + 1), rootController.signal);
     const handleSignal = (signal)=>{
@@ -1189,6 +1288,16 @@ async function runPreparedTestProject(plan) {
             throw error;
         }
     };
+    const stopObservingOperations = observeResourceOperations(rootController.signal, (event)=>{
+        try {
+            journal.append(event);
+        } catch (error) {
+            recordInfrastructureError(new WorkflowPublicationError('write-result', journal.path, error));
+            getDebug('test-runner:operations', {
+                console: true
+            })(`Could not persist operation event: ${String(error)}`);
+        }
+    });
     const announceProject = (prepared, projectIndex)=>{
         const { project } = prepared;
         runInfrastructureCallback(()=>progress(`[project ${projectIndex + 1}/${preparedProjects.length}] ${project.name}`));
@@ -1213,6 +1322,7 @@ async function runPreparedTestProject(plan) {
                 lifecycle
             } : {},
             cases,
+            plans: prepared.plans,
             documents,
             collectionErrors: prepared.collectionErrors
         };
@@ -1331,6 +1441,7 @@ async function runPreparedTestProject(plan) {
     } catch (error) {
         recordInfrastructureError(error);
     } finally{
+        stopObservingOperations();
         process.off('SIGINT', sigint);
         process.off('SIGTERM', sigterm);
     }
@@ -1346,6 +1457,10 @@ async function runPreparedTestProject(plan) {
     const endedAt = new Date();
     const result = {
         schemaVersion: 3,
+        operationLog: {
+            path: journal.path,
+            snapshotAt: endedAt.toISOString()
+        },
         runId,
         startedAt: startedAt.toISOString(),
         endedAt: endedAt.toISOString(),
@@ -1365,7 +1480,6 @@ async function runPreparedTestProject(plan) {
         } : {}
     };
     let completedResult = result;
-    let reportIndex;
     const publish = async (operation, path, callback)=>{
         try {
             await callback();
@@ -1396,7 +1510,6 @@ async function runPreparedTestProject(plan) {
                 overwrite: false,
                 sources: collectTestRunReportSources(completedResult),
                 buildRunnerDump: (index)=>{
-                    reportIndex = index;
                     const dump = buildTestRunReportDump(completedResult, index);
                     const warn = getDebug('test-runner:report-assembler', {
                         console: true
@@ -1414,13 +1527,6 @@ async function runPreparedTestProject(plan) {
     for (const publication of plan.publications)await publish(publication.operation, publication.path, async ()=>{
         await publication.publish(completedResult);
     });
-    await publish('write-result', join(dirname(summaryPath), 'result.md'), async ()=>{
-        const markdownPath = await writeTestResultMarkdown(completedResult, reportIndex);
-        completedResult = {
-            ...completedResult,
-            markdownPath
-        };
-    });
     await writeSummary();
     if (hasInfrastructureError) throw new WorkflowExecutionFailure(completedResult, infrastructureErrors);
     return completedResult;