@travetto/test 8.0.0-alpha.4 → 8.0.0-alpha.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/test",
-  "version": "8.0.0-alpha.4",
+  "version": "8.0.0-alpha.5",
   "type": "module",
   "description": "Declarative test framework",
   "keywords": [
@@ -28,15 +28,15 @@
     "directory": "module/test"
   },
   "dependencies": {
-    "@travetto/registry": "^8.0.0-alpha.4",
-    "@travetto/runtime": "^8.0.0-alpha.4",
-    "@travetto/terminal": "^8.0.0-alpha.4",
-    "@travetto/worker": "^8.0.0-alpha.4",
+    "@travetto/registry": "^8.0.0-alpha.5",
+    "@travetto/runtime": "^8.0.0-alpha.5",
+    "@travetto/terminal": "^8.0.0-alpha.5",
+    "@travetto/worker": "^8.0.0-alpha.5",
     "yaml": "^2.8.2"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.7",
-    "@travetto/transformer": "^8.0.0-alpha.3"
+    "@travetto/cli": "^8.0.0-alpha.8",
+    "@travetto/transformer": "^8.0.0-alpha.4"
   },
   "peerDependenciesMeta": {
     "@travetto/transformer": {

package/src/assert/check.ts

@@ -178,17 +178,13 @@ export class AssertCheck {
   static #onError(
     positive: boolean,
     message: string | undefined,
-    error: unknown,
+    errorValue: unknown,
     missed: Error | undefined,
     shouldThrow: ThrowableError | undefined,
     assertion: CapturedAssertion
   ): void {
-    if (!(error instanceof Error)) {
-      error = new Error(`${error}`);
-    }
-    if (!(error instanceof Error)) {
-      throw error;
-    }
+    const error = errorValue instanceof Error ? errorValue : new Error(`${errorValue}`);
+
     if (positive) {
       missed = new assert.AssertionError({ message: 'Error thrown, but expected no errors' });
       missed.stack = error.stack;

package/src/assert/util.ts

@@ -1,10 +1,9 @@
 import util from 'node:util';
-import path from 'node:path';
 
-import { asFull, type Class, JSONUtil, hasFunction, Runtime, RuntimeIndex, Util } from '@travetto/runtime';
+import { JSONUtil, hasFunction, RuntimeIndex, Util } from '@travetto/runtime';
 
-import type { TestConfig, TestResult } from '../model/test.ts';
-import type { SuiteConfig, SuiteResult } from '../model/suite.ts';
+import type { Assertion, TestConfig } from '../model/test.ts';
+import type { SuiteConfig } from '../model/suite.ts';
 
 const isCleanable = hasFunction<{ toClean(): unknown }>('toClean');
 
@@ -52,79 +51,21 @@ export class AssertUtil {
   /**
    * Generate a suite error given a suite config, and an error
    */
-  static generateSuiteTestFailure(config: { suite: SuiteConfig, test: TestConfig, error: Error, importLocation?: string }): TestResult {
-    const { suite, test, error, importLocation } = config;
+  static generateAssertion(config: { suite: SuiteConfig, test: TestConfig, error: Error, importLocation?: string }): Assertion {
+    const { suite, test, error: errorValue, importLocation } = config;
+    const error = (errorValue.cause && errorValue.cause instanceof Error) ? errorValue.cause : errorValue;
     const testImport = importLocation ?? test.import;
     const position = this.getPositionOfError(error);
     const line = position?.line ?? (testImport === suite.import ? suite.lineStart : 1);
-    const testResult: TestResult = {
-      ...suite.tests[test.methodName],
-      suiteLineStart: suite.lineStart,
-      status: 'errored',
+    return {
+      import: position?.import ?? testImport,
+      methodName: test.methodName,
+      classId: suite.classId,
+      operator: 'throw',
       error,
-      duration: 0,
-      durationTotal: 0,
-      output: [],
-      assertions: [{
-        import: position?.import ?? testImport,
-        methodName: test.methodName,
-        classId: suite.classId,
-        operator: 'throw',
-        error,
-        line,
-        message: error.message.split(/\n/)[0],
-        text: test.methodName
-      }],
+      line,
+      message: error.message.split(/\n/)[0],
+      text: test.methodName
     };
-
-    return testResult;
-  }
-
-  /**
-   * Generate suite failure
-   */
-  static generateSuiteTestFailures(suite: SuiteConfig, error: Error): TestResult[] {
-    const finalError = error.cause instanceof Error ? error.cause : error;
-    return Object.values(suite.tests).map(test => this.generateSuiteTestFailure({ suite, test, error: finalError }));
-  }
-
-  /**
-   * Define import failure as a TestResult
-   */
-  static gernerateImportFailure(importLocation: string, error: Error): { result: TestResult, test: TestConfig, suite: SuiteResult & SuiteConfig } {
-    const name = path.basename(importLocation);
-    const classId = `${RuntimeIndex.getFromImport(importLocation)?.id}#${name}`;
-    const suite = asFull<SuiteConfig & SuiteResult>({
-      class: asFull<Class>({ name }), classId, duration: 0, lineStart: 1, lineEnd: 1, import: importLocation
-    });
-    error.message = error.message.replaceAll(Runtime.mainSourcePath, '.');
-    const result = this.generateSuiteTestFailure({
-      suite,
-      test: {
-        methodName: 'require',
-        classId,
-        import: importLocation,
-        class: suite.class,
-        lineBodyStart: 1,
-        lineStart: 1,
-        lineEnd: 1,
-        skip: false
-      },
-      error
-    });
-    const test: TestConfig = {
-      methodName: 'import',
-      classId,
-      import: importLocation,
-      declarationImport: importLocation,
-      lineStart: 0,
-      lineEnd: 0,
-      lineBodyStart: 0,
-      tags: [],
-      description: 'Import Failure',
-      skip: false,
-      class: undefined!
-    };
-    return { result, test, suite };
   }
 }

package/src/consumer/types/cumulative.ts

@@ -1,7 +1,7 @@
 import type { TestConsumerShape } from '../types.ts';
 import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConfig, TestDiffSource, TestResult } from '../../model/test.ts';
-import type { Counts, SuiteConfig, SuiteResult } from '../../model/suite.ts';
+import type { SuiteConfig, SuiteResult } from '../../model/suite.ts';
 import { DelegatingConsumer } from './delegating.ts';
 import type { SuiteCore } from '../../model/common.ts';
 import { TestModelUtil } from '../../model/util.ts';
@@ -9,7 +9,7 @@ import { TestModelUtil } from '../../model/util.ts';
 type ClassId = string;
 type ImportName = string;
 
-type CumulativeTestResult = Pick<TestResult, 'sourceHash' | 'status' | 'duration'>;
+type CumulativeTestResult = Pick<TestResult, 'sourceHash' | 'status' | 'duration' | 'selfDuration'>;
 type CumulativeSuiteResult = Pick<SuiteCore, 'import' | 'classId' | 'sourceHash'> & {
   tests: Record<string, CumulativeTestResult>;
 };
@@ -37,7 +37,7 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
 
   onTestBefore(config: TestConfig): TestConfig {
     const suite = this.getSuite(config);
-    suite.tests[config.methodName] = { sourceHash: config.sourceHash, status: 'unknown', duration: 0 };
+    suite.tests[config.methodName] = { sourceHash: config.sourceHash, status: 'unknown', duration: 0, selfDuration: 0 };
     return config;
   }
 
@@ -56,21 +56,9 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
   onSuiteAfter(result: SuiteResult): SuiteResult {
     // Reset counts
     const suite = this.getSuite(result);
-    const totals: Counts & { duration: number } = {
-      passed: 0,
-      failed: 0,
-      skipped: 0,
-      errored: 0,
-      unknown: 0,
-      total: 0,
-      duration: 0
-    };
-    for (const test of Object.values(suite.tests)) {
-      totals[test.status] += 1;
-      totals.total += 1;
-      totals.duration += test.duration ?? 0;
-    }
-    return { ...result, ...totals, status: TestModelUtil.countsToTestStatus(totals) };
+    const results = TestModelUtil.buildSummary();
+    TestModelUtil.countTestResult(results, Object.values(suite.tests));
+    return { ...result, ...results, status: TestModelUtil.computeTestStatus(results) };
   }
 
   removeTest(importName: string, classId?: string, methodName?: string): void {

package/src/consumer/types/delegating.ts

@@ -20,6 +20,12 @@ export abstract class DelegatingConsumer implements TestConsumerShape {
     }
   }
 
+  async onTestRunState(state: TestRunState): Promise<void> {
+    for (const consumer of this.#consumers) {
+      await consumer.onTestRunState?.(state);
+    }
+  }
+
   onRemoveEvent(event: TestRemoveEvent): void {
     let result = event;
     if (this.transformRemove) {

package/src/consumer/types/runnable.ts

@@ -22,6 +22,7 @@ export class RunnableTestConsumer extends DelegatingConsumer {
 
   async summarizeAsBoolean(): Promise<boolean> {
     await this.summarize(this.#results?.summary);
-    return (this.#results?.summary.failed ?? 0) <= 0;
+    return (this.#results?.summary.failed ?? 0) <= 0 &&
+      (this.#results?.summary.errored ?? 0) <= 0;
   }
 }

package/src/consumer/types/summarizer.ts

@@ -1,6 +1,7 @@
 import type { SuiteResult } from '../../model/suite.ts';
 import type { TestEvent } from '../../model/event.ts';
 import type { SuitesSummary, TestConsumerShape } from '../types.ts';
+import { TestModelUtil } from '../../model/util.ts';
 
 /**
  * Test Result Collector, combines all results into a single Suite Result
@@ -8,26 +9,13 @@ import type { SuitesSummary, TestConsumerShape } from '../types.ts';
 export class TestResultsSummarizer implements TestConsumerShape {
 
   summary: SuitesSummary = {
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    skipped: 0,
-    unknown: 0,
-    total: 0,
-    duration: 0,
-    suites: [],
-    errors: []
+    ...TestModelUtil.buildSummary(),
+    suites: []
   };
 
   #merge(result: SuiteResult): void {
+    TestModelUtil.countTestResult(this.summary, Object.values(result.tests));
     this.summary.suites.push(result);
-    this.summary.failed += result.failed;
-    this.summary.errored += result.errored;
-    this.summary.passed += result.passed;
-    this.summary.unknown += result.unknown;
-    this.summary.skipped += result.skipped;
-    this.summary.duration += result.duration;
-    this.summary.total += result.total;
   }
 
   /**

package/src/consumer/types/tap-summary.ts

@@ -2,7 +2,7 @@ import { Util, AsyncQueue } from '@travetto/runtime';
 import { StyleUtil, Terminal, TerminalUtil } from '@travetto/terminal';
 
 import type { TestEvent } from '../../model/event.ts';
-import type { TestResult, TestStatus } from '../../model/test.ts';
+import type { TestResult } from '../../model/test.ts';
 
 import type { SuitesSummary, TestConsumerShape, TestRunState } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
@@ -10,6 +10,7 @@ import { TestConsumer } from '../decorator.ts';
 import { TapEmitter } from './tap.ts';
 import { CONSOLE_ENHANCER, type TestResultsEnhancer } from '../enhancer.ts';
 import type { SuiteResult } from '../../model/suite.ts';
+import { TestModelUtil } from '../../model/util.ts';
 
 type Result = {
   key: string;
@@ -31,6 +32,7 @@ export class TapSummaryEmitter implements TestConsumerShape {
   #consumer: TapEmitter;
   #enhancer: TestResultsEnhancer;
   #options?: Record<string, unknown>;
+  #state: TestRunState = {};
 
   constructor(terminal: Terminal = new Terminal(process.stderr)) {
     this.#terminal = terminal;
@@ -87,20 +89,24 @@ export class TapSummaryEmitter implements TestConsumerShape {
     this.#consumer.setOptions(options);
   }
 
+  onTestRunState(state: TestRunState): void {
+    Object.assign(this.#state, state);
+  }
+
   async onStart(state: TestRunState): Promise<void> {
     this.#consumer.onStart();
-    const total: Record<TestStatus | 'count', number> = { errored: 0, failed: 0, passed: 0, skipped: 0, unknown: 0, count: 0 };
+    this.onTestRunState(state);
+
+    const total = TestModelUtil.buildSummary();
     const success = StyleUtil.getStyle({ text: '#e5e5e5', background: '#026020' }); // White on dark green
     const fail = StyleUtil.getStyle({ text: '#e5e5e5', background: '#8b0000' }); // White on dark red
     this.#progress = this.#terminal.streamToBottom(
       Util.mapAsyncIterable(
         this.#results,
         (value) => {
-          total[value.status] += 1;
-          total.count += 1;
+          TestModelUtil.countTestResult(total, [value]);
           const statusLine = `${total.failed} failed, ${total.errored} errored, ${total.skipped} skipped`;
-          return { value: `Tests %idx/%total [${statusLine}] -- ${value.classId}`, total: state.testCount, idx: total.passed };
-
+          return { value: `Tests %idx/%total [${statusLine}] -- ${value.classId}`, total: this.#state.testCount, idx: total.passed };
         },
         TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: (total.failed || total.errored) ? fail : success }) })
       ),
@@ -112,7 +118,7 @@ export class TapSummaryEmitter implements TestConsumerShape {
     if (event.type === 'test' && event.phase === 'after') {
       const { test } = event;
       this.#results.add(test);
-      if (test.status === 'failed') {
+      if (test.status !== 'passed' && test.status !== 'skipped') {
         this.#consumer.onEvent(event);
       }
       const tests = this.#timings.getOrInsert('test', new Map<string, Result>());

package/src/consumer/types/tap.ts

@@ -1,14 +1,17 @@
 import path from 'node:path';
+import { AssertionError } from 'node:assert';
 import { stringify } from 'yaml';
 
 import { Terminal, StyleUtil } from '@travetto/terminal';
-import { TimeUtil, RuntimeIndex, hasToJSON, JSONUtil } from '@travetto/runtime';
+import { TimeUtil, RuntimeIndex } from '@travetto/runtime';
 
 import type { TestEvent } from '../../model/event.ts';
 import type { SuitesSummary, TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
 import { type TestResultsEnhancer, CONSOLE_ENHANCER } from '../enhancer.ts';
 
+const SPACE = ' ';
+
 /**
  * TAP Format consumer
  */
@@ -17,8 +20,8 @@ export class TapEmitter implements TestConsumerShape {
   #count = 0;
   #enhancer: TestResultsEnhancer;
   #terminal: Terminal;
-  #start: number;
   #options?: Record<string, unknown>;
+  #start: number = 0;
 
   constructor(
     terminal = new Terminal(),
@@ -59,16 +62,14 @@ export class TapEmitter implements TestConsumerShape {
    * @param error
    */
   errorToString(error?: Error): string | undefined {
-    if (error && error.name !== 'AssertionError') {
-      if (error instanceof Error) {
-        let out = JSONUtil.toUTF8(hasToJSON(error) ? error.toJSON() : error, { indent: 2 });
-        if (this.#options?.verbose && error.stack) {
-          out = `${out}\n${error.stack}`;
-        }
-        return out;
-      } else {
-        return `${error}`;
-      }
+    if (error instanceof AssertionError) {
+      return;
+    } else if (error instanceof Error) {
+      return error.stack ?
+        error.stack.split(/\n/).slice(0, this.#options?.verbose ? -1 : 5).join('\n') :
+        error.message;
+    } else {
+      return `${error}`;
     }
   }
 
@@ -125,9 +126,10 @@ export class TapEmitter implements TestConsumerShape {
       // Track test result
       let status = `${this.#enhancer.testNumber(++this.#count)} `;
       switch (test.status) {
+        case 'passed': `${this.#enhancer.success('ok')} ${status}`; break;
         case 'skipped': status += ' # SKIP'; break;
-        case 'failed': status = `${this.#enhancer.failure('not ok')} ${status}`; break;
-        default: status = `${this.#enhancer.success('ok')} ${status}`;
+        case 'unknown': break;
+        default: status = `${this.#enhancer.failure('not ok')} ${status}`; break;
       }
       status += header;
 
@@ -138,9 +140,9 @@ export class TapEmitter implements TestConsumerShape {
         case 'errored':
         case 'failed': {
           if (test.error) {
-            const msg = this.errorToString(test.error);
-            if (msg) {
-              this.logMeta({ error: msg });
+            const message = this.errorToString(test.error);
+            if (message) {
+              this.logMeta({ error: message });
             }
           }
           break;
@@ -168,28 +170,22 @@ export class TapEmitter implements TestConsumerShape {
   onSummary(summary: SuitesSummary): void {
     this.log(`${this.#enhancer.testNumber(1)}..${this.#enhancer.testNumber(summary.total)}`);
 
-    if (summary.errors.length) {
-      this.log('---\n');
-      for (const error of summary.errors) {
-        const msg = this.errorToString(error);
-        if (msg) {
-          this.log(this.#enhancer.failure(msg));
-        }
-      }
-    }
-
     const allPassed = !summary.failed && !summary.errored;
 
     this.log([
-      this.#enhancer[allPassed ? 'success' : 'failure']('Results'),
-      `${this.#enhancer.total(summary.passed)}/${this.#enhancer.total(summary.total)},`,
-      allPassed ? 'failed' : this.#enhancer.failure('failed'),
-      `${this.#enhancer.total(summary.failed)}`,
-      allPassed ? 'errored' : this.#enhancer.failure('errored'),
-      `${this.#enhancer.total(summary.errored)}`,
-      'skipped',
-      this.#enhancer.total(summary.skipped),
-      `# (Total Test Time: ${TimeUtil.asClock(summary.duration)}, Total Run Time: ${TimeUtil.asClock(Date.now() - this.#start)})`
-    ].join(' '));
+      this.#enhancer[allPassed ? 'success' : 'failure']('Results'), SPACE,
+      `${this.#enhancer.total(summary.passed)}/${this.#enhancer.total(summary.total)},`, SPACE,
+      allPassed ? 'failed' : this.#enhancer.failure('failed'), SPACE,
+      `${this.#enhancer.total(summary.failed)}`, SPACE,
+      allPassed ? 'errored' : this.#enhancer.failure('errored'), SPACE,
+      `${this.#enhancer.total(summary.errored)}`, SPACE,
+      'skipped', SPACE,
+      this.#enhancer.total(summary.skipped), SPACE,
+      '#', SPACE, '(Timings:', SPACE,
+      'Self=', TimeUtil.asClock(summary.selfDuration), ',', SPACE,
+      'Total=', TimeUtil.asClock(summary.duration), ',', SPACE,
+      'Clock=', TimeUtil.asClock(Date.now() - this.#start),
+      ')',
+    ].join(''));
   }
 }

package/src/consumer/types.ts

@@ -1,24 +1,16 @@
 import type { Class } from '@travetto/runtime';
 
 import type { TestEvent, TestRemoveEvent } from '../model/event.ts';
-import type { Counts, SuiteResult } from '../model/suite.ts';
+import type { ResultsSummary, SuiteResult } from '../model/suite.ts';
 
 /**
  * All suite results
  */
-export interface SuitesSummary extends Counts {
+export interface SuitesSummary extends ResultsSummary {
   /**
    * List of all suites
    */
   suites: SuiteResult[];
-  /**
-   * List of all errors
-   */
-  errors: Error[];
-  /**
-   * Total duration
-   */
-  duration: number;
 }
 
 export type TestRunState = {
@@ -44,6 +36,10 @@ export interface TestConsumerShape {
    * Set options
    */
   setOptions?(options?: Record<string, unknown>): Promise<void> | void;
+  /**
+   * Can directly update the known test run state as needed
+   */
+  onTestRunState?(state: TestRunState): Promise<void> | void;
   /**
    * Listen for start of the test run
    */

package/src/execute/executor.ts

@@ -26,24 +26,6 @@ export class TestExecutor {
     this.#consumer = consumer;
   }
 
-  #onSuiteTestError(result: TestResult, test: TestConfig): void {
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    for (const assertion of result.assertions) {
-      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion });
-    }
-    this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
-  }
-
-  #recordSuiteErrors(suiteConfig: SuiteConfig, suiteResult: SuiteResult, errors: TestResult[]): void {
-    for (const test of errors) {
-      if (!suiteResult.tests[test.methodName]) {
-        this.#onSuiteTestError(test, suiteConfig.tests[test.methodName]);
-        suiteResult.errored += 1;
-        suiteResult.total += 1;
-      }
-    }
-  }
-
   /**
    * Raw execution, runs the method and then returns any thrown errors as the result.
    *
@@ -73,96 +55,43 @@ export class TestExecutor {
     }
   }
 
-  #skipTest(test: TestConfig, result: SuiteResult): void {
-    // Mark test start
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    result.skipped += 1;
-    result.total += 1;
-    this.#consumer.onEvent({
-      type: 'test',
-      phase: 'after',
-      test: {
-        ...test,
-        suiteLineStart: result.lineStart,
-        assertions: [], duration: 0, durationTotal: 0, output: [], status: 'skipped'
-      }
-    });
-  }
-
-  /**
-   * An empty suite result based on a suite config
-   */
-  createSuiteResult(suite: SuiteConfig, override?: Partial<SuiteResult>): SuiteResult {
-    return {
-      passed: 0,
-      failed: 0,
-      errored: 0,
-      skipped: 0,
-      unknown: 0,
-      total: 0,
-      status: 'unknown',
-      lineStart: suite.lineStart,
-      lineEnd: suite.lineEnd,
-      import: suite.import,
-      classId: suite.classId,
-      sourceHash: suite.sourceHash,
-      duration: 0,
-      tests: {},
-      ...override
-    };
-  }
-
   /**
    * Execute the test, capture output, assertions and promises
    */
-  async executeTest(test: TestConfig, suite: SuiteConfig): Promise<TestResult> {
+  async executeTest(test: TestConfig, suite: SuiteConfig, override?: Partial<TestResult>): Promise<TestResult> {
+
+    const result = TestModelUtil.createTestResult(suite, test, override);
 
     // Mark test start
     this.#consumer.onEvent({ type: 'test', phase: 'before', test });
 
-    const startTime = Date.now();
-
-    const result: TestResult = {
-      methodName: test.methodName,
-      description: test.description,
-      classId: test.classId,
-      tags: test.tags,
-      suiteLineStart: suite.lineStart,
-      lineStart: test.lineStart,
-      lineEnd: test.lineEnd,
-      lineBodyStart: test.lineBodyStart,
-      import: test.import,
-      declarationImport: test.declarationImport,
-      sourceHash: test.sourceHash,
-      status: 'unknown',
-      assertions: [],
-      duration: 0,
-      durationTotal: 0,
-      output: [],
-    };
 
     // Emit every assertion as it occurs
-    const getAssertions = AssertCapture.collector(test, asrt =>
-      this.#consumer.onEvent({
-        type: 'assertion',
-        phase: 'after',
-        assertion: asrt
-      })
-    );
+    const getAssertions = AssertCapture.collector(test, item =>
+      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: item }));
 
     const consoleCapture = new ConsoleCapture().start(); // Capture all output from transpiled code
 
-    // Run method and get result
-    const error = await this.#executeTestMethod(test);
-    const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+    // Already finished
+    if (result.status !== 'unknown') {
+      if (result.error) {
+        result.assertions.push(AssertUtil.generateAssertion({ suite, test, error: result.error }));
+      }
+      for (const item of result.assertions ?? []) { AssertCapture.add(item); }
+    } else {
+      // Run method and get result
+      const startTime = Date.now();
+      const error = await this.#executeTestMethod(test);
+      const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+      result.status = status;
+      result.selfDuration = Date.now() - startTime;
+      if (finalError) {
+        result.error = finalError;
+      }
+    }
 
-    Object.assign(result, {
-      status,
-      output: consoleCapture.end(),
-      assertions: getAssertions(),
-      duration: Date.now() - startTime,
-      ...(finalError ? { error: finalError } : {})
-    });
+    result.output = consoleCapture.end();
+    result.assertions = getAssertions();
 
     // Mark completion
     this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
@@ -179,14 +108,17 @@ export class TestExecutor {
 
     const shouldSkip = await this.#shouldSkip(suite, suite.instance);
 
+    const result: SuiteResult = TestModelUtil.createSuiteResult(suite);
+
     if (shouldSkip) {
       this.#consumer.onEvent({
         phase: 'after', type: 'suite',
-        suite: this.createSuiteResult(suite, {
+        suite: {
+          ...result,
           status: 'skipped',
           skipped: tests.length,
           total: tests.length
-        })
+        }
       });
     }
 
@@ -194,69 +126,80 @@ export class TestExecutor {
       return;
     }
 
-    const result: SuiteResult = this.createSuiteResult(suite);
+    const manager = new TestPhaseManager(suite);
+    const originalEnv = { ...process.env };
+    const startTime = Date.now();
+    const testResultOverrides: Record<string, Partial<TestResult>> = {};
+
     const validTestMethodNames = new Set(tests.map(t => t.methodName));
     const testConfigs = Object.fromEntries(
       Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key))
     );
 
-    const startTime = Date.now();
-
     // Mark suite start
     this.#consumer.onEvent({ phase: 'before', type: 'suite', suite: { ...suite, tests: testConfigs } });
 
-    const manager = new TestPhaseManager(suite);
-
-    const originalEnv = { ...process.env };
-
     try {
       // Handle the BeforeAll calls
       await manager.startPhase('all');
+    } catch (someError) {
+      const suiteError = await manager.onError('all', someError);
+      for (const method of validTestMethodNames) {
+        testResultOverrides[method] ??= { status: 'errored', error: suiteError };
+      }
+    }
 
-      const suiteEnv = { ...process.env };
+    const suiteEnv = { ...process.env };
 
-      for (const test of tests ?? suite.tests) {
-        if (await this.#shouldSkip(test, suite.instance)) {
-          this.#skipTest(test, result);
-          continue;
-        }
+    for (const test of tests) {
+      // Reset env before each test
+      process.env = { ...suiteEnv };
 
-        // Reset env before each test
-        process.env = { ...suiteEnv };
+      const testStart = Date.now();
+      const testResultOverride = (testResultOverrides[test.methodName] ??= {});
 
-        const testStart = Date.now();
-        try {
+      if (await this.#shouldSkip(test, suite.instance)) {
+        testResultOverride.status = 'skipped';
+      }
 
-          // Handle BeforeEach
-          await manager.startPhase('each');
+      try {
+        // Handle BeforeEach
+        testResultOverride.status || await manager.startPhase('each');
+      } catch (someError) {
+        const testError = await manager.onError('each', someError);
+        testResultOverride.error = testError;
+        testResultOverride.status = 'errored';
+      }
 
-          // Run test
-          const testResult = await this.executeTest(test, suite);
-          result.tests[testResult.methodName] = testResult;
-          result[testResult.status]++;
-          result.total += 1;
+      // Run test
+      const testResult = await this.executeTest(test, suite, testResultOverride);
 
-          // Handle after each
-          await manager.endPhase('each');
-          testResult.durationTotal = Date.now() - testStart;
-        } catch (testError) {
-          const errors = await manager.errorPhase('each', testError, suite, test);
-          this.#recordSuiteErrors(suite, result, errors);
-        }
+      // Handle after each
+      try {
+        testResultOverride.status || await manager.endPhase('each');
+      } catch (testError) {
+        if (!(testError instanceof Error)) { throw testError; };
+        console.error('Failed to properly shutdown test', testError.message);
       }
 
+      result.tests[testResult.methodName] = testResult;
+      testResult.duration = Date.now() - testStart;
+      TestModelUtil.countTestResult(result, [testResult]);
+    }
+
+    try {
       // Handle after all
       await manager.endPhase('all');
     } catch (suiteError) {
-      const errors = await manager.errorPhase('all', suiteError, suite);
-      this.#recordSuiteErrors(suite, result, errors);
+      if (!(suiteError instanceof Error)) { throw suiteError; };
+      console.error('Failed to properly shutdown test', suiteError.message);
     }
 
     // Restore env
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-    result.status = TestModelUtil.countsToTestStatus(result);
+    result.status = TestModelUtil.computeTestStatus(result);
 
     // Mark suite complete
     this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
@@ -265,19 +208,13 @@ export class TestExecutor {
   /**
    * Handle executing a suite's test/tests based on command line inputs
    */
-  async execute(run: TestRun): Promise<void> {
+  async execute(run: TestRun, singleFile?: boolean): Promise<void> {
     try {
       await Runtime.importFrom(run.import);
     } catch (error) {
-      if (!(error instanceof Error)) {
-        throw error;
-      }
+      if (!(error instanceof Error)) { throw error; }
+      const suite = TestModelUtil.createImportErrorSuiteResult(run);
       console.error(error);
-
-      // Fire import failure as a test failure for each test in the suite
-      const { result, test, suite } = AssertUtil.gernerateImportFailure(run.import, error);
-      this.#consumer.onEvent({ type: 'suite', phase: 'before', suite });
-      this.#onSuiteTestError(result, test);
       this.#consumer.onEvent({ type: 'suite', phase: 'after', suite });
       return;
     }
@@ -291,6 +228,11 @@ export class TestExecutor {
       console.warn('Unable to find suites for ', run);
     }
 
+    if (singleFile) {
+      const testCount = suites.reduce((acc, suite) => acc + suite.tests.length, 0);
+      this.#consumer.onTestRunState?.({ testCount });
+    }
+
     for (const { suite, tests } of suites) {
       await this.executeSuite(suite, tests);
     }

package/src/execute/phase.ts

@@ -1,9 +1,7 @@
-import { describeFunction, Env, TimeUtil } from '@travetto/runtime';
+import { Env, TimeUtil } from '@travetto/runtime';
 
 import type { SuiteConfig, SuitePhase } from '../model/suite.ts';
-import { AssertUtil } from '../assert/util.ts';
 import { Barrier } from './barrier.ts';
-import type { TestConfig, TestResult } from '../model/test.ts';
 
 const TEST_PHASE_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_PHASE_TIMEOUT.value ?? 15000, 'ms');
 
@@ -34,9 +32,7 @@ export class TestPhaseManager {
       error = await Barrier.awaitOperation(TEST_PHASE_TIMEOUT, async () => handler[phase]?.(this.#suite.instance));
 
       if (error) {
-        const toThrow = new Error(phase, { cause: error });
-        Object.assign(toThrow, { import: describeFunction(handler.constructor) ?? undefined });
-        throw toThrow;
+        throw error;
       }
     }
   }
@@ -58,21 +54,13 @@ export class TestPhaseManager {
   }
 
   /**
-   * Handles if an error occurs during a phase, ensuring that we attempt to end the phase and then return the appropriate test results for the failure
+   * Handle an error during phase operation
    */
-  async errorPhase(phase: 'all' | 'each', error: unknown, suite: SuiteConfig, test?: TestConfig): Promise<TestResult[]> {
-    try { await this.endPhase(phase); } catch { }
-    if (!(error instanceof Error)) { throw error; }
-
-    // Don't propagate our own errors
-    if (error.message === 'afterAll' || error.message === 'afterEach') {
-      return [];
-    }
-
-    if (test) {
-      return [AssertUtil.generateSuiteTestFailure({ suite, error, test })];
-    } else {
-      return AssertUtil.generateSuiteTestFailures(suite, error);
+  async onError(phase: 'all' | 'each', error: unknown): Promise<Error> {
+    if (!(error instanceof Error)) {
+      await this.endPhase(phase).catch(() => { });
+      throw error;
     }
+    return error;
   }
 }

package/src/execute/run.ts

@@ -222,7 +222,7 @@ export class RunUtil {
     }
 
     if (runs.length === 1) {
-      await new TestExecutor(consumer).execute(runs[0]);
+      await new TestExecutor(consumer).execute(runs[0], true);
     } else {
       await WorkPool.run(
         run => buildStandardTestManager(consumer, run),

package/src/model/suite.ts

@@ -34,29 +34,35 @@ export interface SuiteConfig extends SuiteCore {
 }
 
 /**
- * All counts for the suite summary
+ * Test Counts
  */
-export interface Counts {
+export interface ResultsSummary {
+  /** Passing Test Count  */
   passed: number;
+  /** Skipped Test Count  */
   skipped: number;
+  /** Failed Test Count  */
   failed: number;
+  /** Errored Test Count  */
   errored: number;
+  /** Unknown Test Count  */
   unknown: number;
+  /** Total Test Count  */
   total: number;
+  /** Test Self Execution Duration  */
+  selfDuration: number;
+  /** Total Duration */
+  duration: number;
 }
 
 /**
  * Results of a suite run
  */
-export interface SuiteResult extends Counts, SuiteCore {
+export interface SuiteResult extends ResultsSummary, SuiteCore {
   /**
    * All test results
    */
   tests: Record<string, TestResult>;
-  /**
-   * Suite duration
-   */
-  duration: number;
   /**
    * Overall status
    */

package/src/model/test.ts

@@ -105,13 +105,13 @@ export interface TestResult extends TestCore {
    */
   assertions: Assertion[];
   /**
-   * Duration for the test
+   * Self Execution Duration
    */
-  duration: number;
+  selfDuration: number;
   /**
    * Total duration including before/after
    */
-  durationTotal: number;
+  duration: number;
   /**
    * Logging output
    */

package/src/model/util.ts

@@ -1,14 +1,97 @@
-import type { Counts } from './suite.ts';
-import type { TestStatus } from './test.ts';
+import path from 'node:path';
+
+import { asFull, RuntimeIndex } from '@travetto/runtime';
+
+import type { ResultsSummary, SuiteConfig, SuiteResult } from './suite.ts';
+import type { TestConfig, TestResult, TestRun, TestStatus } from './test.ts';
 
 export class TestModelUtil {
-  static countsToTestStatus(counts: Counts): TestStatus {
+  static computeTestStatus(summary: ResultsSummary): TestStatus {
     switch (true) {
-      case counts.errored > 0: return 'errored';
-      case counts.failed > 0: return 'failed';
-      case counts.skipped > 0: return 'skipped';
-      case counts.unknown > 0: return 'unknown';
+      case summary.errored > 0: return 'errored';
+      case summary.failed > 0: return 'failed';
+      case summary.skipped > 0: return 'skipped';
+      case summary.unknown > 0: return 'unknown';
       default: return 'passed';
     }
   }
+
+  static buildSummary(): ResultsSummary {
+    return { passed: 0, failed: 0, skipped: 0, errored: 0, unknown: 0, total: 0, duration: 0, selfDuration: 0 };
+  }
+
+  static countTestResult<T extends ResultsSummary>(summary: T, tests: Pick<TestResult, 'status' | 'selfDuration' | 'duration'>[]): T {
+    for (const test of tests) {
+      summary[test.status] += 1;
+      summary.total += 1;
+      summary.selfDuration += (test.selfDuration ?? 0);
+      summary.duration += (test.duration ?? 0);
+    }
+    return summary;
+  }
+
+
+  /**
+   * An empty suite result based on a suite config
+   */
+  static createSuiteResult(suite: SuiteConfig, override?: Partial<SuiteResult>): SuiteResult {
+    return {
+      ...TestModelUtil.buildSummary(),
+      status: 'unknown',
+      lineStart: suite.lineStart,
+      lineEnd: suite.lineEnd,
+      import: suite.import,
+      classId: suite.classId,
+      sourceHash: suite.sourceHash,
+      tests: {},
+      duration: 0,
+      selfDuration: 0,
+      ...override
+    };
+  }
+
+  /**
+   * An empty test result based on a suite and test config
+   */
+  static createTestResult(suite: SuiteConfig, test: TestConfig, override?: Partial<TestResult>): TestResult {
+    return {
+      methodName: test.methodName,
+      description: test.description,
+      classId: test.classId,
+      tags: test.tags,
+      suiteLineStart: suite.lineStart,
+      lineStart: test.lineStart,
+      lineEnd: test.lineEnd,
+      lineBodyStart: test.lineBodyStart,
+      import: test.import,
+      declarationImport: test.declarationImport,
+      sourceHash: test.sourceHash,
+      status: 'unknown',
+      assertions: [],
+      duration: 0,
+      selfDuration: 0,
+      output: [],
+      ...override
+    };
+  }
+
+  static createImportErrorSuiteResult(run: TestRun): SuiteResult {
+    const name = path.basename(run.import);
+    const classId = `${RuntimeIndex.getFromImport(run.import)?.id}#${name}`;
+    const common = { classId, duration: 0, lineStart: 1, lineEnd: 1, import: run.import } as const;
+    return asFull<SuiteResult>({
+      ...common,
+      status: 'errored', errored: 1,
+      tests: {
+        impport: asFull<TestResult>({
+          ...common,
+          status: 'errored',
+          assertions: [{
+            ...common, line: common.lineStart,
+            methodName: 'import', operator: 'import', text: `Failed to import ${run.import}`,
+          }]
+        })
+      }
+    });
+  }
 }

package/src/worker/child.ts

@@ -20,9 +20,7 @@ export class TestChildWorker extends IpcChannel<TestRun> {
       await operation();
       this.send(type); // Respond
     } catch (error) {
-      if (!(error instanceof Error)) {
-        throw error;
-      }
+      if (!(error instanceof Error)) { throw error; }
       // Mark as errored out
       this.send(type, JSONUtil.cloneForTransmit(error));
     }