@midscene/test 1.12.5 → 1.12.6-beta-20260909034232.0

package/dist/es/test-cli.mjs

@@ -6,8 +6,10 @@ import { require as api_require } from "tsx/cjs/api";
 import { tsImport } from "tsx/esm/api";
 import { createHash, randomUUID } from "node:crypto";
 import { setMaxListeners } from "node:events";
+import { TestRunReportAssembler } from "@midscene/core/report";
 import { globSync } from "tinyglobby";
 import { JSON_SCHEMA, load } from "js-yaml";
+import { getDebug } from "@midscene/shared/logger";
 var __webpack_modules__ = {
     "node:fs" (module) {
         module.exports = __rspack_external_node_fs_5ea92f0c;
@@ -985,6 +987,30 @@ async function executeNode(step, execute, phase, stepIndex, options) {
         parentSignal?.removeEventListener('abort', abortFromParent);
     }
 }
+function createNodeReportCollector(node) {
+    const traces = new Map();
+    let accepting = true;
+    return {
+        collector: Object.freeze({
+            addTrace (trace) {
+                if (!accepting) throw new NodeExecutionError(node, new Error('Report traces can only be added while the current Step is running.'));
+                if (!trace || 'object' != typeof trace || 'midscene-execution' !== trace.type) throw new NodeExecutionError(node, new TypeError('Report trace type must be "midscene-execution".'));
+                if ('string' != typeof trace.executionId || 0 === trace.executionId.trim().length) throw new NodeExecutionError(node, new TypeError('Report trace executionId must be non-empty.'));
+                const normalized = Object.freeze({
+                    type: 'midscene-execution',
+                    executionId: trace.executionId
+                });
+                traces.set(`${normalized.type}:${normalized.executionId}`, normalized);
+            }
+        }),
+        close () {
+            accepting = false;
+            return Object.freeze([
+                ...traces.values()
+            ]);
+        }
+    };
+}
 async function parseNodeInput(node, input) {
     if (!node.inputSchema) return input;
     const parsed = await node.inputSchema.safeParseAsync(input);
@@ -994,32 +1020,47 @@ async function parseNodeInput(node, input) {
 async function executeStep(step, node, target, context, execution = {}) {
     const phase = 'case' === target.scope ? target.case.phase : target.document.phase;
     const stepIndex = 'case' === target.scope ? target.case.stepIndex : target.document.stepIndex;
-    return executeNode(step, async (signal)=>{
-        const input = await parseNodeInput(node, step.input);
-        signal.throwIfAborted();
-        const common = {
-            input,
-            $: step.meta,
-            signal,
-            context,
-            onTeardown: (teardown)=>{
-                if (!execution.onTeardown) throw new NodeExecutionError(step.node, new Error('The current execution scope cannot register teardown.'));
-                execution.onTeardown(step.node, teardown);
+    const reportCollector = createNodeReportCollector(step.node);
+    let result;
+    try {
+        result = await executeNode(step, async (signal)=>{
+            const input = await parseNodeInput(node, step.input);
+            signal.throwIfAborted();
+            const common = {
+                input,
+                $: step.meta,
+                signal,
+                context,
+                report: reportCollector.collector,
+                onTeardown: (teardown)=>{
+                    if (!execution.onTeardown) throw new NodeExecutionError(step.node, new Error('The current execution scope cannot register teardown.'));
+                    execution.onTeardown(step.node, teardown);
+                }
+            };
+            return 'case' === target.scope ? node.execute({
+                ...common,
+                scope: 'case',
+                case: target.case
+            }) : node.execute({
+                ...common,
+                scope: 'document',
+                document: target.document
+            });
+        }, phase, stepIndex, {
+            parentSignal: execution.signal,
+            defaultTimeoutMs: execution.defaultTimeoutMs
+        });
+    } finally{
+        const traces = reportCollector.close();
+        if (traces.length > 0 && result) result = {
+            ...result,
+            report: {
+                traces
             }
         };
-        return 'case' === target.scope ? node.execute({
-            ...common,
-            scope: 'case',
-            case: target.case
-        }) : node.execute({
-            ...common,
-            scope: 'document',
-            document: target.document
-        });
-    }, phase, stepIndex, {
-        parentSignal: execution.signal,
-        defaultTimeoutMs: execution.defaultTimeoutMs
-    });
+    }
+    if (!result) throw new NodeExecutionError(step.node, new Error('Step execution completed without a result.'));
+    return result;
 }
 const reportPathsFromTeardown = (result)=>{
     if (void 0 === result) return [];
@@ -1654,6 +1695,419 @@ function collectWorkflowDocument(source, options) {
         cases
     };
 }
+const MAX_DEPTH = 6;
+const MAX_STRING_LENGTH = 2000;
+const MAX_ARRAY_ITEMS = 50;
+const MAX_OBJECT_KEYS = 50;
+const MAX_SERIALIZED_BYTES = 65536;
+const TRUNCATED_VALUE = '[Truncated: display value exceeded 64 KiB]';
+const normalizedSecretKeys = [
+    'password',
+    'passwd',
+    'secret',
+    'token',
+    'authorization',
+    'cookie',
+    'apikey'
+];
+const normalizedKey = (key)=>key.toLowerCase().replace(/[^a-z0-9]/g, '');
+const isSecretKey = (key)=>{
+    const normalized = normalizedKey(key);
+    return normalizedSecretKeys.some((secret)=>normalized.includes(secret));
+};
+const objectPath = (parent, key)=>/^[A-Za-z_$][\w$]*$/.test(key) ? `${parent}.${key}` : `${parent}[${JSON.stringify(key)}]`;
+const stringByteLength = (value)=>Buffer.byteLength(value, 'utf8');
+function sanitizeReportText(value, maximumLength = MAX_STRING_LENGTH) {
+    return value.length <= maximumLength ? value : `${value.slice(0, maximumLength)}… [truncated]`;
+}
+function sanitizeReportValue(value) {
+    const truncatedPaths = new Set();
+    const redactedPaths = new Set();
+    const ancestors = new WeakSet();
+    let remainingBytes = MAX_SERIALIZED_BYTES;
+    const consume = (path, text)=>{
+        remainingBytes -= stringByteLength(text);
+        if (remainingBytes >= 0) return true;
+        truncatedPaths.add(path);
+        return false;
+    };
+    const visit = (current, path, depth)=>{
+        if (remainingBytes <= 0) {
+            truncatedPaths.add(path);
+            return '[Truncated: size limit]';
+        }
+        if (null === current || 'boolean' == typeof current) {
+            consume(path, String(current));
+            return current;
+        }
+        if ('string' == typeof current) {
+            if (current.length > MAX_STRING_LENGTH) {
+                truncatedPaths.add(path);
+                const shortened = `${current.slice(0, MAX_STRING_LENGTH)}… [truncated]`;
+                consume(path, shortened);
+                return shortened;
+            }
+            consume(path, current);
+            return current;
+        }
+        if ('number' == typeof current) {
+            if (Number.isFinite(current)) {
+                consume(path, String(current));
+                return current;
+            }
+            truncatedPaths.add(path);
+            return `[Unsupported number: ${String(current)}]`;
+        }
+        if ('bigint' == typeof current) {
+            truncatedPaths.add(path);
+            return `[Unsupported bigint: ${String(current)}]`;
+        }
+        if (void 0 === current || 'function' == typeof current || 'symbol' == typeof current) {
+            truncatedPaths.add(path);
+            return `[Unsupported ${typeof current}]`;
+        }
+        if (depth >= MAX_DEPTH) {
+            truncatedPaths.add(path);
+            return '[Truncated: maximum depth]';
+        }
+        if (current instanceof Date) {
+            const date = Number.isNaN(current.getTime()) ? '[Invalid Date]' : current.toISOString();
+            consume(path, date);
+            return date;
+        }
+        if (ancestors.has(current)) {
+            truncatedPaths.add(path);
+            return '[Unsupported circular value]';
+        }
+        if (Array.isArray(current)) {
+            ancestors.add(current);
+            const output = [];
+            const itemCount = Math.min(current.length, MAX_ARRAY_ITEMS);
+            for(let index = 0; index < itemCount; index += 1){
+                if (remainingBytes <= 0) break;
+                output.push(visit(current[index], `${path}[${index}]`, depth + 1));
+            }
+            if (current.length > output.length) truncatedPaths.add(path);
+            ancestors.delete(current);
+            return output;
+        }
+        const prototype = Object.getPrototypeOf(current);
+        if (prototype !== Object.prototype && null !== prototype) {
+            truncatedPaths.add(path);
+            const constructorName = current.constructor?.name ?? 'object';
+            return `[Unsupported object: ${constructorName}]`;
+        }
+        ancestors.add(current);
+        const output = {};
+        const keys = Object.keys(current);
+        for (const key of keys.slice(0, MAX_OBJECT_KEYS)){
+            const childPath = objectPath(path, key);
+            if (!consume(childPath, key)) break;
+            if (isSecretKey(key)) {
+                output[key] = '[REDACTED]';
+                redactedPaths.add(childPath);
+            } else output[key] = visit(current[key], childPath, depth + 1);
+            if (remainingBytes <= 0) break;
+        }
+        if (keys.length > Object.keys(output).length) truncatedPaths.add(path);
+        ancestors.delete(current);
+        return output;
+    };
+    const sanitized = visit(value, '$', 0);
+    const serialized = JSON.stringify(sanitized);
+    if (void 0 === serialized || stringByteLength(serialized) > MAX_SERIALIZED_BYTES) return {
+        value: TRUNCATED_VALUE,
+        truncatedPaths: [
+            '$'
+        ]
+    };
+    return {
+        value: sanitized,
+        ...truncatedPaths.size > 0 ? {
+            truncatedPaths: [
+                ...truncatedPaths
+            ]
+        } : {},
+        ...redactedPaths.size > 0 ? {
+            redactedPaths: [
+                ...redactedPaths
+            ]
+        } : {}
+    };
+}
+function sanitizeReportError(error) {
+    if (error instanceof Error) {
+        const workflowLike = error;
+        return {
+            name: sanitizeReportText(error.name || 'Error', 200),
+            message: sanitizeReportText(error.message || String(error)),
+            ...'string' == typeof workflowLike.code ? {
+                code: sanitizeReportText(workflowLike.code, 200)
+            } : {},
+            ...void 0 === workflowLike.details ? {} : {
+                details: sanitizeReportValue(workflowLike.details)
+            }
+        };
+    }
+    return {
+        name: 'Error',
+        message: sanitizeReportText(String(error))
+    };
+}
+const warnReport = getDebug('test-runner:report-assembler', {
+    console: true
+});
+const exactMidsceneDetailNodes = new Set([
+    'aiAct',
+    'aiAssert',
+    'recordToReport'
+]);
+const scopedExecutionKey = (scopeId, executionId)=>`${scopeId}\u0000${executionId}`;
+const reportSourcesForScope = (context, scopeId)=>context.sourcesByScope.get(scopeId) ?? [];
+const addDiagnostic = (context, diagnostic)=>{
+    context.diagnostics.push(diagnostic);
+    const key = JSON.stringify([
+        diagnostic.code,
+        diagnostic.scopeId,
+        diagnostic.executionId
+    ]);
+    if (!context.warnedDiagnostics.has(key)) {
+        context.warnedDiagnostics.add(key);
+        warnReport(diagnostic.message);
+    }
+};
+const buildStep = (step, scopeId, context)=>{
+    const details = [];
+    const unresolvedExecutionIds = [];
+    for (const trace of step.report?.traces ?? []){
+        const previousScope = context.tracedScopeByExecution.get(trace.executionId);
+        if (previousScope && previousScope !== scopeId) throw new Error(`Agent execution ${trace.executionId} was traced by multiple test scopes (${previousScope}, ${scopeId}).`);
+        context.tracedScopeByExecution.set(trace.executionId, scopeId);
+        const detail = context.detailsByScopeAndExecution.get(scopedExecutionKey(scopeId, trace.executionId));
+        if (detail) details.push(detail);
+        else unresolvedExecutionIds.push(trace.executionId);
+    }
+    let agentDetailDiagnostic;
+    if (unresolvedExecutionIds.length > 0) {
+        agentDetailDiagnostic = `Agent detail could not be resolved for ${unresolvedExecutionIds.join(', ')}.`;
+        const sources = reportSourcesForScope(context, scopeId);
+        for (const executionId of unresolvedExecutionIds)addDiagnostic(context, {
+            level: 'warning',
+            code: 'agent-detail-unresolved',
+            message: `Agent detail could not be resolved for execution ${executionId} in test scope ${scopeId}.`,
+            scopeId,
+            executionId,
+            ...sources.length > 0 ? {
+                sourcePaths: sources.map((source)=>source.sourcePath)
+            } : {}
+        });
+    } else if (0 === details.length && exactMidsceneDetailNodes.has(step.node)) {
+        agentDetailDiagnostic = 'This Agent did not expose a stable execution reference for the Step.';
+        addDiagnostic(context, {
+            level: 'warning',
+            code: 'agent-detail-unavailable',
+            message: `No stable Agent execution reference was captured for ${step.node} in test scope ${scopeId}.`,
+            scopeId,
+            ...reportSourcesForScope(context, scopeId).length > 0 ? {
+                sourcePaths: reportSourcesForScope(context, scopeId).map((source)=>source.sourcePath)
+            } : {}
+        });
+    }
+    const prompt = 'string' == typeof step.input?.prompt ? step.input.prompt : void 0;
+    return {
+        id: `${scopeId}:${step.phase}:${step.stepIndex}`,
+        phase: step.phase,
+        stepIndex: step.stepIndex,
+        node: step.node,
+        ...prompt ? {
+            title: sanitizeReportText(prompt, 240)
+        } : {},
+        status: step.status,
+        continuedAfterError: step.continuedAfterError,
+        startedAt: step.startedAt,
+        endedAt: step.endedAt,
+        durationMs: step.durationMs,
+        input: sanitizeReportValue(step.input),
+        ...step.output ? {
+            output: {
+                ...void 0 === step.output.summary ? {} : {
+                    summary: sanitizeReportText(step.output.summary)
+                },
+                ...void 0 === step.output.data ? {} : {
+                    data: sanitizeReportValue(step.output.data)
+                }
+            }
+        } : {},
+        ...step.error ? {
+            error: sanitizeReportError(step.error)
+        } : {},
+        ...details.length > 0 ? {
+            agentDetails: details
+        } : {},
+        ...agentDetailDiagnostic ? {
+            agentDetailDiagnostic
+        } : {}
+    };
+};
+const buildAttempt = (attempt, context)=>({
+        attemptId: attempt.runId,
+        attemptIndex: attempt.attemptIndex,
+        status: attempt.status,
+        startedAt: attempt.startedAt,
+        endedAt: attempt.endedAt,
+        durationMs: attempt.durationMs,
+        beforeEach: attempt.beforeEach.map((step)=>buildStep(step, attempt.runId, context)),
+        steps: attempt.steps.map((step)=>buildStep(step, attempt.runId, context)),
+        afterEach: attempt.afterEach.map((step)=>buildStep(step, attempt.runId, context)),
+        ...attempt.teardownErrors?.length ? {
+            teardownErrors: attempt.teardownErrors.map(sanitizeReportError)
+        } : {},
+        ...reportSourcesForScope(context, attempt.runId).length > 0 ? {
+            scopeReportIds: reportSourcesForScope(context, attempt.runId).map((source)=>source.reportId)
+        } : {}
+    });
+const buildCase = (outcome, context)=>({
+        caseId: outcome.caseId,
+        name: outcome.name,
+        caseIndex: outcome.caseIndex,
+        status: outcome.status,
+        ...outcome.notRunReason ? {
+            notRunReason: outcome.notRunReason
+        } : {},
+        attempts: (outcome.attempts ?? (outcome.run ? [
+            outcome.run
+        ] : [])).map((attempt)=>buildAttempt(attempt, context))
+    });
+const buildDocument = (documentId, document, cases, context)=>{
+    const sourcePath = document?.sourcePath ?? cases[0]?.sourcePath ?? '<unknown>';
+    const status = document?.status ?? (cases.some((item)=>'success' !== item.status) ? 'failed' : 'success');
+    return {
+        documentId,
+        sourcePath,
+        status,
+        ...document ? {
+            startedAt: document.startedAt,
+            endedAt: document.endedAt,
+            durationMs: document.durationMs
+        } : {},
+        beforeAll: (document?.beforeAll ?? []).map((step)=>buildStep(step, document.documentRunId, context)),
+        cases: cases.map((outcome)=>buildCase(outcome, context)),
+        afterAll: (document?.afterAll ?? []).map((step)=>buildStep(step, document.documentRunId, context)),
+        ...document?.teardownErrors?.length ? {
+            teardownErrors: document.teardownErrors.map(sanitizeReportError)
+        } : {},
+        ...document && reportSourcesForScope(context, document.documentRunId).length > 0 ? {
+            scopeReportIds: reportSourcesForScope(context, document.documentRunId).map((source)=>source.reportId)
+        } : {}
+    };
+};
+const buildProject = (project, context)=>{
+    const documentById = new Map(project.documents.map((document)=>[
+            document.documentId,
+            document
+        ]));
+    const casesByDocumentId = new Map();
+    for (const outcome of project.cases){
+        const current = casesByDocumentId.get(outcome.documentId) ?? [];
+        current.push(outcome);
+        casesByDocumentId.set(outcome.documentId, current);
+    }
+    const documentIds = new Set([
+        ...project.documents.map((document)=>document.documentId),
+        ...project.cases.map((outcome)=>outcome.documentId)
+    ]);
+    return {
+        projectId: project.projectId,
+        name: project.name,
+        platform: project.platform,
+        status: project.status,
+        retry: project.retry,
+        ...project.lifecycle ? {
+            lifecycle: {
+                status: project.lifecycle.status,
+                startedAt: project.lifecycle.startedAt,
+                endedAt: project.lifecycle.endedAt,
+                durationMs: project.lifecycle.durationMs,
+                ...project.lifecycle.setupError ? {
+                    setupError: sanitizeReportError(project.lifecycle.setupError)
+                } : {},
+                ...project.lifecycle.teardownErrors?.length ? {
+                    teardownErrors: project.lifecycle.teardownErrors.map(sanitizeReportError)
+                } : {}
+            }
+        } : {},
+        documents: [
+            ...documentIds
+        ].map((documentId)=>buildDocument(documentId, documentById.get(documentId), casesByDocumentId.get(documentId) ?? [], context)),
+        collectionErrors: project.collectionErrors.map((collectionError)=>({
+                sourcePath: collectionError.sourcePath,
+                error: sanitizeReportError(collectionError.error)
+            }))
+    };
+};
+function collectTestRunReportSources(result) {
+    const sources = [];
+    for (const project of result.projects){
+        for (const document of project.documents)for (const sourcePath of document.reportPaths ?? [])sources.push({
+            scopeId: document.documentRunId,
+            sourcePath
+        });
+        for (const outcome of project.cases)for (const attempt of outcome.attempts ?? (outcome.run ? [
+            outcome.run
+        ] : []))for (const sourcePath of attempt.reportPaths ?? [])sources.push({
+            scopeId: attempt.runId,
+            sourcePath
+        });
+    }
+    return sources;
+}
+function buildTestRunReportDump(result, index) {
+    const sourcesByScope = new Map();
+    const detailsByScopeAndExecution = new Map();
+    for (const source of index.sources){
+        const scopeSources = sourcesByScope.get(source.scopeId) ?? [];
+        scopeSources.push(source);
+        sourcesByScope.set(source.scopeId, scopeSources);
+        for (const executionId of source.executionIds)detailsByScopeAndExecution.set(scopedExecutionKey(source.scopeId, executionId), {
+            reportId: source.reportId,
+            executionId
+        });
+    }
+    const context = {
+        diagnostics: [],
+        sourcesByScope,
+        detailsByScopeAndExecution,
+        tracedScopeByExecution: new Map(),
+        warnedDiagnostics: new Set()
+    };
+    for (const source of index.sources)if (0 === source.executionIds.length) addDiagnostic(context, {
+        level: 'warning',
+        code: 'source-without-executions',
+        message: `Agent report contains no executions: ${source.sourcePath}`,
+        scopeId: source.scopeId,
+        sourcePaths: [
+            source.sourcePath
+        ]
+    });
+    const projects = result.projects.map((project)=>buildProject(project, context));
+    return {
+        schemaVersion: 1,
+        kind: 'test-runner',
+        runId: result.runId,
+        status: result.status,
+        startedAt: result.startedAt,
+        endedAt: result.endedAt,
+        durationMs: result.durationMs,
+        summary: {
+            ...result.summary
+        },
+        metrics: index.metrics,
+        projects,
+        ...context.diagnostics.length > 0 ? {
+            diagnostics: context.diagnostics
+        } : {}
+    };
+}
 const writeJson = (path, value)=>{
     (0, external_node_fs_.mkdirSync)((0, external_node_path_.dirname)(path), {
         recursive: true
@@ -1709,6 +2163,9 @@ const writeTestProjectRunResult = (options)=>{
         } : {},
         factsRoot: '.',
         reportDir: relativeToSummary(result.summaryPath, result.reportDir),
+        ...result.reportPath ? {
+            report: relativeToSummary(result.summaryPath, result.reportPath)
+        } : {},
         summary: result.summary,
         projects: result.projects.map((project)=>({
                 projectId: project.projectId,
@@ -2196,7 +2653,24 @@ async function runTestProject(options = {}) {
         } : {},
         result
     });
-    return result;
+    const reportPath = new TestRunReportAssembler().assemble({
+        outputDir: reportDir,
+        reportFileName: `test-run-${runId}`,
+        sources: collectTestRunReportSources(result),
+        buildRunnerDump: (index)=>buildTestRunReportDump(result, index)
+    });
+    const completedResult = {
+        ...result,
+        reportPath
+    };
+    writeTestProjectRunResult({
+        projectRoot,
+        ...configPath ? {
+            configPath
+        } : {},
+        result: completedResult
+    });
+    return completedResult;
 }
 const parseTestCliArgs = (args, cwd = process.cwd())=>{
     const command = 'nodes' === args[0] ? args[0] : void 0;
@@ -2288,6 +2762,7 @@ async function runTestCli(args, io = defaultCliIO) {
         io.log(`midscene-test: ${result.summary.passed}/${result.summary.total} cases passed, ${result.summary.failed} failed, ${result.summary.notRun} not run`);
         io.log(`Results: ${result.resultDir}`);
         io.log(`Summary: ${result.summaryPath}`);
+        if (result.reportPath) io.log(`Report: ${result.reportPath}`);
         return result.exitCode;
     } catch (error) {
         io.error(error instanceof Error ? error.message : String(error));