@travetto/test 5.0.0-rc.0 → 5.0.0-rc.10

package/src/consumer/types/execution.ts

@@ -2,7 +2,7 @@ import { ChildCommChannel } from '@travetto/worker';
 
 import { TestEvent } from '../../model/event';
 import { TestConsumer } from '../types';
-import { ErrorUtil } from '../error';
+import { SerializeUtil } from '../serialize';
 import { Consumable } from '../registry';
 
 /**
@@ -12,8 +12,6 @@ import { Consumable } from '../registry';
 export class ExecutionEmitter extends ChildCommChannel<TestEvent> implements TestConsumer {
 
   onEvent(event: TestEvent): void {
-    const out = { ...event };
-    ErrorUtil.serializeTestErrors(out);
-    this.send(event.type, out);
+    this.send(event.type, JSON.parse(SerializeUtil.serializeToJSON(event)));
   }
 }

package/src/consumer/types/runnable.ts

@@ -1,62 +1,33 @@
-import { TestConsumer, TestRunState } from '../types';
+import { TestConsumer } from '../types';
 import { TestResultsSummarizer } from './summarizer';
 import { TestConsumerRegistry } from '../registry';
 import { TestEvent } from '../../model/event';
+import { DelegatingConsumer } from './delegating';
 
 /**
  * Test consumer with support for multiple nested consumers, and summarization
  */
-export class RunnableTestConsumer implements TestConsumer {
+export class RunnableTestConsumer extends DelegatingConsumer {
   /**
    * Build a runnable test consumer given a format or a full consumer
    */
   static async get(consumer: string | TestConsumer): Promise<RunnableTestConsumer> {
-    return new RunnableTestConsumer(await TestConsumerRegistry.getInstance(consumer));
+    return new RunnableTestConsumer([await TestConsumerRegistry.getInstance(consumer)]);
   }
 
-  #consumers: TestConsumer[];
-  #results: TestResultsSummarizer | undefined;
+  #results?: TestResultsSummarizer;
 
-  constructor(...consumers: TestConsumer[]) {
-    this.#consumers = consumers;
-    for (const c of consumers) {
-      if (!this.#results && c.onSummary) { // If expecting summary
-        this.#results = new TestResultsSummarizer();
-      }
-      c.onEvent = c.onEvent.bind(c);
-    }
+  constructor(consumers: TestConsumer[]) {
+    super(consumers);
+    this.#results = consumers.find(x => !!x.onSummary) ? new TestResultsSummarizer() : undefined;
   }
 
-  async onStart(state: TestRunState): Promise<void> {
-    for (const c of this.#consumers) {
-      await c.onStart?.(state);
-    }
-  }
-
-  onEvent(e: TestEvent): void {
-    if (this.#results) {
-      this.#results.onEvent(e);
-    }
-    for (const c of this.#consumers) {
-      c.onEvent(e);
-    }
-  }
-
-  async summarize(): Promise<TestResultsSummarizer | undefined> {
-    if (this.#results) {
-      for (const c of this.#consumers) {
-        await c.onSummary?.(this.#results.summary);
-      }
-      return this.#results;
-    }
+  onEventDone(e: TestEvent): void {
+    this.#results?.onEvent(e);
   }
 
   async summarizeAsBoolean(): Promise<boolean> {
-    const result = await this.summarize();
-    if (result) {
-      return result.summary.failed <= 0;
-    } else {
-      return true;
-    }
+    await this.summarize(this.#results?.summary);
+    return (this.#results?.summary.failed ?? 0) <= 0;
   }
 }

package/src/consumer/types/tap-streamed.ts

@@ -1,6 +1,5 @@
-import { Util } from '@travetto/base';
+import { Util, AsyncQueue } from '@travetto/runtime';
 import { StyleUtil, Terminal, TerminalUtil } from '@travetto/terminal';
-import { WorkQueue } from '@travetto/worker';
 
 import { TestEvent } from '../../model/event';
 import { TestResult } from '../../model/test';
@@ -17,7 +16,7 @@ import { TapEmitter } from './tap';
 export class TapStreamedEmitter implements TestConsumer {
 
   #terminal: Terminal;
-  #results = new WorkQueue<TestResult>();
+  #results = new AsyncQueue<TestResult>();
   #progress: Promise<unknown> | undefined;
   #consumer: TapEmitter;
 
@@ -30,16 +29,20 @@ export class TapStreamedEmitter implements TestConsumer {
     this.#consumer.onStart();
 
     let failed = 0;
-    const succ = StyleUtil.getStyle({ text: '#e5e5e5', background: '#026020' }); // White on dark green
+    let skipped = 0;
+    let completed = 0;
+    const success = StyleUtil.getStyle({ text: '#e5e5e5', background: '#026020' }); // White on dark green
     const fail = StyleUtil.getStyle({ text: '#e5e5e5', background: '#8b0000' }); // White on dark red
     this.#progress = this.#terminal.streamToBottom(
       Util.mapAsyncItr(
         this.#results,
         (value, idx) => {
           failed += (value.status === 'failed' ? 1 : 0);
-          return { value: `Tests %idx/%total [${failed} failed] -- ${value.classId}`, total: state.testCount, idx };
+          skipped += (value.status === 'skipped' ? 1 : 0);
+          completed += (value.status !== 'skipped' ? 1 : 0);
+          return { value: `Tests %idx/%total [${failed} failed, ${skipped} skipped] -- ${value.classId}`, total: state.testCount, idx: completed };
         },
-        TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: failed ? fail : succ }) })
+        TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: failed ? fail : success }) })
       ),
       { minDelay: 100 }
     );

package/src/consumer/types/tap.ts

@@ -1,12 +1,11 @@
-import { RuntimeIndex } from '@travetto/manifest';
 import { Terminal } from '@travetto/terminal';
-import { AppError, TimeUtil } from '@travetto/base';
+import { AppError, TimeUtil, Runtime, RuntimeIndex } from '@travetto/runtime';
 import { stringify } from 'yaml';
 
 import { TestEvent } from '../../model/event';
 import { SuitesSummary, TestConsumer } from '../types';
 import { Consumable } from '../registry';
-import { ErrorUtil } from '../error';
+import { SerializeUtil } from '../serialize';
 import { TestResultsEnhancer, CONSOLE_ENHANCER } from '../enhancer';
 
 /**
@@ -67,11 +66,12 @@ export class TapEmitter implements TestConsumer {
         let subCount = 0;
         for (const asrt of test.assertions) {
           const text = asrt.message ? `${asrt.text} (${this.#enhancer.failure(asrt.message)})` : asrt.text;
+          const pth = RuntimeIndex.getFromImport(asrt.import)!.sourceFile.replace(Runtime.mainSourcePath, '.');
           let subMessage = [
             this.#enhancer.assertNumber(++subCount),
             '-',
             this.#enhancer.assertDescription(text),
-            `${this.#enhancer.assertFile(asrt.file.replace(RuntimeIndex.mainModule.sourcePath, '.'))}:${this.#enhancer.assertLine(asrt.line)}`
+            `${this.#enhancer.assertFile(pth)}:${this.#enhancer.assertLine(asrt.line)}`
           ].join(' ');
 
           if (asrt.error) {
@@ -102,7 +102,7 @@ export class TapEmitter implements TestConsumer {
       // Handle error
       if (test.status === 'failed') {
         if (test.error && test.error.name !== 'AssertionError') {
-          const err = ErrorUtil.deserializeError(test.error);
+          const err = SerializeUtil.deserializeError(test.error);
           this.logMeta({ error: err instanceof AppError ? err.toJSON() : err });
         }
       }

package/src/consumer/types/xunit.ts

@@ -2,6 +2,8 @@ import { Writable } from 'node:stream';
 
 import { stringify } from 'yaml';
 
+import { RuntimeIndex } from '@travetto/runtime';
+
 import { TestEvent } from '../../model/event';
 import { SuitesSummary, TestConsumer } from '../types';
 import { Consumable } from '../registry';
@@ -85,7 +87,7 @@ export class XunitEmitter implements TestConsumer {
     failures="${suite.failed}"
     errors="${suite.failed}"
     skipped="${suite.skipped}"
-    file="${suite.file}"
+    file="${RuntimeIndex.getFromImport(suite.import)!.sourceFile}"
   >
       ${testBodies.join('\n')}
       </testsuite>
@@ -101,7 +103,7 @@ export class XunitEmitter implements TestConsumer {
     this.#stream.write(`
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites
-  name="${summary.suites.length ? summary.suites[0].file : 'nameless'}"
+  name="${summary.suites.length ? RuntimeIndex.getFromImport(summary.suites[0].import)?.sourceFile : 'nameless'}"
   time="${summary.duration}"
   tests="${summary.total}"
   failures="${summary.failed}"

package/src/decorator/suite.ts

@@ -1,5 +1,4 @@
-import { RuntimeIndex } from '@travetto/manifest';
-import { Class, ClassInstance } from '@travetto/base';
+import { castTo, Class, ClassInstance, describeFunction } from '@travetto/runtime';
 
 import { SuiteRegistry } from '../registry/suite';
 import { SuiteConfig } from '../model/suite';
@@ -24,17 +23,16 @@ export function Suite(description?: string | Partial<SuiteConfig>, ...rest: Part
     Object.assign(extra, r);
   }
 
-  // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-  const decorator = ((target: Class) => {
+  const dec = (target: Class): typeof target => {
     const cfg = { description: descriptionString, ...extra };
-    if (RuntimeIndex.getFunctionMetadata(target)?.abstract) {
+    if (describeFunction(target).abstract) {
       cfg.skip = true;
     }
     SuiteRegistry.register(target, cfg);
     return target;
-  }) as ClassDecorator;
+  };
 
-  return decorator;
+  return castTo(dec);
 }
 
 function listener(phase: SuitePhase) {

package/src/decorator/test.ts

@@ -1,4 +1,4 @@
-import { ClassInstance } from '@travetto/base';
+import { ClassInstance } from '@travetto/runtime';
 
 import { SuiteRegistry } from '../registry/suite';
 import { TestConfig } from '../model/test';
@@ -15,6 +15,7 @@ export function AssertCheck(): MethodDecorator {
  * @param description The test description
  * @augments `@travetto/test:Test`
  * @augments `@travetto/test:AssertCheck`
+ * @augments `@travetto/runtime:DebugBreak`
  */
 export function Test(): MethodDecorator;
 export function Test(...rest: Partial<TestConfig>[]): MethodDecorator;

package/src/execute/console.ts

@@ -1,6 +1,6 @@
 import util from 'node:util';
 
-import { ConsoleEvent, ConsoleListener, ConsoleManager } from '@travetto/base';
+import { ConsoleEvent, ConsoleListener, ConsoleManager } from '@travetto/runtime';
 
 /**
  * Console capturer.  Hooks into the Console manager, and collects the

package/src/execute/executor.ts

@@ -1,13 +1,11 @@
 import { AssertionError } from 'node:assert';
-import path from 'node:path';
 
-import { RuntimeIndex, RuntimeContext } from '@travetto/manifest';
-import { Env, TimeUtil } from '@travetto/base';
+import { Env, TimeUtil, Runtime, castTo } from '@travetto/runtime';
 import { Barrier, ExecutionError } from '@travetto/worker';
 
 import { SuiteRegistry } from '../registry/suite';
-import { TestConfig, TestResult } from '../model/test';
-import { SuiteConfig, SuiteResult } from '../model/suite';
+import { TestConfig, TestResult, TestRun } from '../model/test';
+import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite';
 import { TestConsumer } from '../consumer/types';
 import { AssertCheck } from '../assert/check';
 import { AssertCapture } from '../assert/capture';
@@ -23,12 +21,42 @@ const TEST_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_TIMEOUT.val) ?? 5000;
  */
 export class TestExecutor {
 
+  #consumer: TestConsumer;
+  #testFilter: (config: TestConfig) => boolean;
+
+  constructor(consumer: TestConsumer, testFilter?: (config: TestConfig) => boolean) {
+    this.#consumer = consumer;
+    this.#testFilter = testFilter || ((): boolean => true);
+  }
+
+  /**
+   * Handles communicating a suite-level error
+   * @param failure
+   * @param withSuite
+   */
+  #onSuiteFailure(failure: SuiteFailure, triggerSuite?: boolean): void {
+    if (triggerSuite) {
+      this.#consumer.onEvent({ type: 'suite', phase: 'before', suite: failure.suite });
+    }
+
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test: failure.test });
+    this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: failure.assert });
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: failure.testResult });
+
+    if (triggerSuite) {
+      this.#consumer.onEvent({
+        type: 'suite', phase: 'after',
+        suite: { ...castTo(failure.suite), failed: 1, passed: 0, total: 1, skipped: 0 }
+      });
+    }
+  }
+
   /**
    * Raw execution, runs the method and then returns any thrown errors as the result.
    *
    * This method should never throw under any circumstances.
    */
-  static async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
+  async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
     const suite = SuiteRegistry.get(test.class);
 
     // Ensure all the criteria below are satisfied before moving forward
@@ -40,8 +68,7 @@ export class TestExecutor {
 
         try {
           await pCap.run(() =>
-            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-            (suite.instance as Record<string, Function>)[test.methodName]()
+            castTo<Record<string, Function>>(suite.instance)[test.methodName]()
           );
         } finally {
           process.env = env; // Restore
@@ -55,35 +82,27 @@ export class TestExecutor {
   /**
    * Determining if we should skip
    */
-  static async #skip(cfg: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
-    if (cfg.skip !== undefined) {
-      if (typeof cfg.skip === 'boolean' ? cfg.skip : await cfg.skip(inst)) {
-        return true;
-      }
+  async #shouldSkip(cfg: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
+    if ('methodName' in cfg && !this.#testFilter(cfg)) {
+      return true;
+    }
+
+    if (typeof cfg.skip === 'function' ? await cfg.skip(inst) : cfg.skip) {
+      return true;
     }
   }
 
-  /**
-   * Fail an entire file, marking the whole file as failed
-   */
-  static failFile(consumer: TestConsumer, file: string, err: Error): void {
-    const name = path.basename(file);
-    const classId = RuntimeIndex.getId(file, name);
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const suite = { class: { name }, classId, duration: 0, lineStart: 1, lineEnd: 1, file, } as SuiteConfig & SuiteResult;
-    err.message = err.message.replaceAll(RuntimeIndex.mainModule.sourcePath, '.');
-    const res = AssertUtil.generateSuiteError(suite, 'require', err);
-    consumer.onEvent({ type: 'suite', phase: 'before', suite });
-    consumer.onEvent({ type: 'test', phase: 'before', test: res.testConfig });
-    consumer.onEvent({ type: 'assertion', phase: 'after', assertion: res.assert });
-    consumer.onEvent({ type: 'test', phase: 'after', test: res.testResult });
-    consumer.onEvent({ type: 'suite', phase: 'after', suite: { ...suite, failed: 1, passed: 0, total: 1, skipped: 0 } });
+  #skipTest(test: TestConfig, result: SuiteResult): void {
+    // Mark test start
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
+    result.skipped++;
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: { ...test, assertions: [], duration: 0, durationTotal: 0, output: {}, status: 'skipped' } });
   }
 
   /**
    * An empty suite result based on a suite config
    */
-  static createSuiteResult(suite: SuiteConfig): SuiteResult {
+  createSuiteResult(suite: SuiteConfig): SuiteResult {
     return {
       passed: 0,
       failed: 0,
@@ -91,7 +110,7 @@ export class TestExecutor {
       total: 0,
       lineStart: suite.lineStart,
       lineEnd: suite.lineEnd,
-      file: suite.file,
+      import: suite.import,
       classId: suite.classId,
       duration: 0,
       tests: []
@@ -101,22 +120,22 @@ export class TestExecutor {
   /**
    * Execute the test, capture output, assertions and promises
    */
-  static async executeTest(consumer: TestConsumer, test: TestConfig, suite: SuiteConfig): Promise<TestResult> {
+  async executeTest(test: TestConfig): Promise<TestResult> {
 
     // Mark test start
-    consumer.onEvent({ type: 'test', phase: 'before', test });
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
 
     const startTime = Date.now();
 
     const result: TestResult = {
       methodName: test.methodName,
-      module: RuntimeContext.main.name,
       description: test.description,
       classId: test.classId,
       lineStart: test.lineStart,
       lineEnd: test.lineEnd,
       lineBodyStart: test.lineBodyStart,
-      file: test.file,
+      import: test.import,
+      sourceImport: test.sourceImport,
       status: 'skipped',
       assertions: [],
       duration: 0,
@@ -124,16 +143,12 @@ export class TestExecutor {
       output: {},
     };
 
-    if (await this.#skip(test, suite.instance)) {
-      return result;
-    }
-
     // Emit every assertion as it occurs
-    const getAssertions = AssertCapture.collector(test, assrt =>
-      consumer.onEvent({
+    const getAssertions = AssertCapture.collector(test, asrt =>
+      this.#consumer.onEvent({
         type: 'assertion',
         phase: 'after',
-        assertion: assrt
+        assertion: asrt
       })
     );
 
@@ -165,47 +180,27 @@ export class TestExecutor {
     });
 
     // Mark completion
-    consumer.onEvent({ type: 'test', phase: 'after', test: result });
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
 
     return result;
   }
 
   /**
-   * Execute a single test within a suite
+   * Execute an entire suite
    */
-  static async executeSuiteTest(consumer: TestConsumer, suite: SuiteConfig, test: TestConfig): Promise<void> {
-    const result: SuiteResult = this.createSuiteResult(suite);
-
-    const mgr = new TestPhaseManager(consumer, suite, result);
-
-    try {
-      await mgr.startPhase('all');
-      const skip = await this.#skip(test, suite.instance);
-      if (!skip) {
-        await mgr.startPhase('each');
-      }
-      await this.executeTest(consumer, test, suite);
-      if (!skip) {
-        await mgr.endPhase('each');
-      }
-      await mgr.endPhase('all');
-    } catch (err) {
-      await mgr.onError(err);
+  async executeSuite(suite: SuiteConfig, tests: TestConfig[]): Promise<void> {
+    if (!tests.length || await this.#shouldSkip(suite, suite.instance)) {
+      return;
     }
-  }
 
-  /**
-   * Execute an entire suite
-   */
-  static async executeSuite(consumer: TestConsumer, suite: SuiteConfig): Promise<SuiteResult> {
     const result: SuiteResult = this.createSuiteResult(suite);
 
     const startTime = Date.now();
 
     // Mark suite start
-    consumer.onEvent({ phase: 'before', type: 'suite', suite });
+    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite });
 
-    const mgr = new TestPhaseManager(consumer, suite, result);
+    const mgr = new TestPhaseManager(suite, result, e => this.#onSuiteFailure(e));
 
     const originalEnv = { ...process.env };
 
@@ -215,28 +210,30 @@ export class TestExecutor {
 
       const suiteEnv = { ...process.env };
 
-      for (const test of suite.tests) {
+      for (const test of tests ?? suite.tests) {
+        if (await this.#shouldSkip(test, suite.instance)) {
+          this.#skipTest(test, result);
+          continue;
+        }
+
         // Reset env before each test
         process.env = { ...suiteEnv };
+
         const testStart = Date.now();
-        const skip = await this.#skip(test, suite.instance);
-        if (!skip) {
-          // Handle BeforeEach
-          await mgr.startPhase('each');
-        }
+
+        // Handle BeforeEach
+        await mgr.startPhase('each');
 
         // Run test
-        const ret = await this.executeTest(consumer, test, suite);
+        const ret = await this.executeTest(test);
         result[ret.status]++;
-
-        if (!skip) {
-          result.tests.push(ret);
-        }
+        result.tests.push(ret);
 
         // Handle after each
         await mgr.endPhase('each');
         ret.durationTotal = Date.now() - testStart;
       }
+
       // Handle after all
       await mgr.endPhase('all');
     } catch (err) {
@@ -247,31 +244,23 @@ export class TestExecutor {
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-
-    // Mark suite complete
-    consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
-
     result.total = result.passed + result.failed;
 
-    return result;
+    // Mark suite complete
+    this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
   }
 
   /**
    * Handle executing a suite's test/tests based on command line inputs
    */
-  static async execute(consumer: TestConsumer, file: string, ...args: string[]): Promise<void> {
-
-    file = path.resolve(file);
-
-    const entry = RuntimeIndex.getEntry(file)!;
-
+  async execute(run: TestRun): Promise<void> {
     try {
-      await import(entry.import);
+      await Runtime.importFrom(run.import);
     } catch (err) {
       if (!(err instanceof Error)) {
         throw err;
       }
-      this.failFile(consumer, file, err);
+      this.#onSuiteFailure(AssertUtil.gernerateImportFailure(run.import, err));
       return;
     }
 
@@ -279,19 +268,10 @@ export class TestExecutor {
     await SuiteRegistry.init();
 
     // Convert inbound arguments to specific tests to run
-    const params = SuiteRegistry.getRunParams(file, ...args);
+    const suites = SuiteRegistry.getSuiteTests(run);
 
-    // If running specific suites
-    if ('suites' in params) {
-      for (const suite of params.suites) {
-        if (!(await this.#skip(suite, suite.instance)) && suite.tests.length) {
-          await this.executeSuite(consumer, suite);
-        }
-      }
-    } else if (params.test) { // If running a single test
-      await this.executeSuiteTest(consumer, params.suite, params.test);
-    } else { // Running the suite
-      await this.executeSuite(consumer, params.suite);
+    for (const { suite, tests } of suites) {
+      await this.executeSuite(suite, tests);
     }
   }
 }

package/src/execute/phase.ts

@@ -1,12 +1,12 @@
 import { Barrier } from '@travetto/worker';
-import { Env, TimeUtil } from '@travetto/base';
+import { Env, TimeUtil } from '@travetto/runtime';
 
-import { TestConsumer } from '../consumer/types';
-import { SuiteConfig, SuiteResult } from '../model/suite';
+import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite';
 import { AssertUtil } from '../assert/util';
-import { TestResult } from '../model/test';
 
-class TestBreakout extends Error { }
+class TestBreakout extends Error {
+  source?: Error;
+}
 
 const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.val) ?? 15000;
 
@@ -17,27 +17,14 @@ const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.val) ??
  */
 export class TestPhaseManager {
   #progress: ('all' | 'each')[] = [];
-  #consumer: TestConsumer;
   #suite: SuiteConfig;
   #result: SuiteResult;
+  #onSuiteFailure: (fail: SuiteFailure) => void;
 
-  constructor(consumer: TestConsumer, suite: SuiteConfig, result: SuiteResult) {
-    this.#consumer = consumer;
+  constructor(suite: SuiteConfig, result: SuiteResult, onSuiteFailure: (fail: SuiteFailure) => void) {
     this.#suite = suite;
     this.#result = result;
-  }
-
-  /**
-   * Create the appropriate events when a suite has an error
-   */
-  async triggerSuiteError(methodName: string, error: Error): Promise<TestResult> {
-    const bad = AssertUtil.generateSuiteError(this.#suite, methodName, error);
-
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test: bad.testConfig });
-    this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: bad.assert });
-    this.#consumer.onEvent({ type: 'test', phase: 'after', test: bad.testResult });
-
-    return bad.testResult;
+    this.#onSuiteFailure = onSuiteFailure;
   }
 
   /**
@@ -57,10 +44,9 @@ export class TestPhaseManager {
       }
     }
     if (error) {
-      const res = await this.triggerSuiteError(`[[${phase}]]`, error);
-      this.#result.tests.push(res);
-      this.#result.failed++;
-      throw new TestBreakout();
+      const tbo = new TestBreakout(`[[${phase}]]`);
+      tbo.source = error;
+      throw tbo;
     }
   }
 
@@ -96,10 +82,14 @@ export class TestPhaseManager {
 
     this.#progress = [];
 
-    if (!(err instanceof TestBreakout)) {
-      const res = await this.triggerSuiteError('all', err);
-      this.#result.tests.push(res);
-      this.#result.failed++;
-    }
+    const failure = AssertUtil.generateSuiteFailure(
+      this.#suite,
+      err instanceof TestBreakout ? err.message : 'all',
+      err instanceof TestBreakout ? err.source! : err
+    );
+
+    this.#onSuiteFailure(failure);
+    this.#result.tests.push(failure.testResult);
+    this.#result.failed++;
   }
 }