@travetto/test 8.0.0-alpha.2 → 8.0.0-alpha.21

package/src/decorator/suite.ts

@@ -1,4 +1,4 @@
-import { castTo, type Class, type ClassInstance, getClass } from '@travetto/runtime';
+import { type Class, type ClassInstance, castTo, getClass } from '@travetto/runtime';
 
 import type { SuiteConfig } from '../model/suite.ts';
 import { SuiteRegistryIndex } from '../registry/registry-index.ts';
@@ -18,7 +18,7 @@ export function Suite(description?: string | Partial<SuiteConfig>, ...rest: Part
     SuiteRegistryIndex.getForRegister(cls).register(
       ...(typeof description !== 'string' && description ? [description] : []),
       ...rest,
-      ...(typeof description === 'string' ? [{ description }] : []),
+      ...(typeof description === 'string' ? [{ description }] : [])
     );
     return cls;
   };
@@ -76,4 +76,4 @@ export function AfterEach() {
     });
     return descriptor;
   };
-}
+}

package/src/decorator/test.ts

@@ -27,10 +27,12 @@ export function Test(...rest: Partial<TestConfig>[]): MethodDecorator;
 export function Test(description: string, ...rest: Partial<TestConfig>[]): MethodDecorator;
 export function Test(description?: string | Partial<TestConfig>, ...rest: Partial<TestConfig>[]): MethodDecorator {
   return (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => {
-    SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(property, descriptor.value,
-      ...(typeof description !== 'string' && description) ? [description] : [],
+    SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(
+      property,
+      descriptor.value,
+      ...(typeof description !== 'string' && description ? [description] : []),
       ...rest,
-      ...(typeof description === 'string') ? [{ description }] : []
+      ...(typeof description === 'string' ? [{ description }] : [])
     );
     return descriptor;
   };
@@ -58,4 +60,4 @@ export function Timeout(ms: number): MethodDecorator {
     SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(property, descriptor.value, { timeout: ms });
     return descriptor;
   };
-}
+}

package/src/execute/barrier.ts

@@ -1,5 +1,5 @@
-import { isPromise } from 'node:util/types';
 import { createHook, executionAsyncId } from 'node:async_hooks';
+import { isPromise } from 'node:util/types';
 
 import { type TimeSpan, TimeUtil, Util } from '@travetto/runtime';
 
@@ -11,7 +11,7 @@ export class Barrier {
   /**
    * Track timeout
    */
-  static timeout(duration: number | TimeSpan, operation: string = 'Operation'): { promise: Promise<void>, resolve: () => unknown } {
+  static timeout(duration: number | TimeSpan, operation: string = 'Operation'): { promise: Promise<void>; resolve: () => unknown } {
     const resolver = Promise.withResolvers<void>();
     const durationMs = TimeUtil.duration(duration, 'ms');
     let timeout: NodeJS.Timeout;
@@ -22,25 +22,31 @@ export class Barrier {
       timeout = setTimeout(() => resolver.reject(new TimeoutError(msg)), durationMs).unref();
     }
 
-    resolver.promise.finally(() => { clearTimeout(timeout); });
+    resolver.promise.finally(() => {
+      clearTimeout(timeout);
+    });
     return resolver;
   }
 
   /**
    * Track uncaught error
    */
-  static uncaughtErrorPromise(): { promise: Promise<void>, resolve: () => unknown } {
+  static uncaughtErrorPromise(): { promise: Promise<void>; resolve: () => unknown } {
     const uncaught = Promise.withResolvers<void>();
-    const onError = (error: Error): void => { Util.queueMacroTask().then(() => uncaught.reject(error)); };
+    const onError = (error: Error): void => {
+      Util.queueMacroTask().then(() => uncaught.reject(error));
+    };
     UNCAUGHT_ERR_EVENTS.map(key => process.on(key, onError));
-    uncaught.promise.finally(() => { UNCAUGHT_ERR_EVENTS.map(key => process.off(key, onError)); });
+    uncaught.promise.finally(() => {
+      UNCAUGHT_ERR_EVENTS.map(key => process.off(key, onError));
+    });
     return uncaught;
   }
 
   /**
    * Promise capturer
    */
-  static capturePromises(): { start: () => Promise<void>, finish: () => Promise<void>, cleanup: () => void } {
+  static capturePromises(): { start: () => Promise<void>; finish: () => Promise<void>; cleanup: () => void } {
     const pending = new Map<number, Promise<unknown>>();
     let id: number = 0;
 
@@ -90,7 +96,7 @@ export class Barrier {
       let capturedError: Error | undefined;
       const opProm = operation().then(() => promises.finish());
 
-      await Promise.race([opProm, uncaught.promise, timer.promise]).catch(error => capturedError ??= error);
+      await Promise.race([opProm, uncaught.promise, timer.promise]).catch(error => (capturedError ??= error));
 
       return capturedError;
     } finally {
@@ -99,4 +105,4 @@ export class Barrier {
       uncaught.resolve();
     }
   }
-}
+}

package/src/execute/console.ts

@@ -1,6 +1,7 @@
 import util from 'node:util';
 
 import { type ConsoleEvent, type ConsoleListener, ConsoleManager } from '@travetto/runtime';
+
 import type { TestLog } from '../model/test.ts';
 
 /**
@@ -21,9 +22,7 @@ export class ConsoleCapture implements ConsoleListener {
   log({ args, scope: _, ...rest }: ConsoleEvent): void {
     this.out.push({
       ...rest,
-      message: args
-        .map((arg => typeof arg === 'string' ? arg : util.inspect(arg, false, 5)))
-        .join(' ')
+      message: args.map(arg => (typeof arg === 'string' ? arg : util.inspect(arg, false, 5))).join(' ')
     });
   }
 
@@ -33,4 +32,4 @@ export class ConsoleCapture implements ConsoleListener {
     ConsoleManager.set(ConsoleCapture.#listener);
     return result;
   }
-}
+}

package/src/execute/executor.ts

@@ -1,17 +1,17 @@
-import { Env, TimeUtil, Runtime, castTo, classConstruct } from '@travetto/runtime';
 import { Registry } from '@travetto/registry';
+import { castTo, classConstruct, Env, Runtime, TimeUtil } from '@travetto/runtime';
 
-import type { TestConfig, TestResult, TestRun } from '../model/test.ts';
-import type { SuiteConfig, SuiteResult } from '../model/suite.ts';
-import type { TestConsumerShape } from '../consumer/types.ts';
-import { AssertCheck } from '../assert/check.ts';
 import { AssertCapture } from '../assert/capture.ts';
-import { ConsoleCapture } from './console.ts';
-import { TestPhaseManager } from './phase.ts';
+import { AssertCheck } from '../assert/check.ts';
 import { AssertUtil } from '../assert/util.ts';
-import { Barrier } from './barrier.ts';
-import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+import type { TestConsumerShape } from '../consumer/types.ts';
+import type { SuiteConfig, SuiteResult } from '../model/suite.ts';
+import type { TestConfig, TestResult, TestRun } from '../model/test.ts';
 import { TestModelUtil } from '../model/util.ts';
+import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+import { Barrier } from './barrier.ts';
+import { ConsoleCapture } from './console.ts';
+import { TestPhaseManager } from './phase.ts';
 
 const TEST_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_TIMEOUT.value || 5000, 'ms');
 
@@ -19,45 +19,24 @@ const TEST_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_TIMEOUT.value || 5000, 'ms')
  * Support execution of the tests
  */
 export class TestExecutor {
-
   #consumer: TestConsumerShape;
 
   constructor(consumer: TestConsumerShape) {
     this.#consumer = consumer;
   }
 
-  #onSuiteTestError(result: TestResult, test: TestConfig): void {
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    for (const assertion of result.assertions) {
-      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion });
-    }
-    this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
-  }
-
-  #recordSuiteErrors(suiteConfig: SuiteConfig, suiteResult: SuiteResult, errors: TestResult[]): void {
-    for (const test of errors) {
-      if (!suiteResult.tests[test.methodName]) {
-        this.#onSuiteTestError(test, suiteConfig.tests[test.methodName]);
-        suiteResult.errored += 1;
-        suiteResult.total += 1;
-      }
-    }
-  }
-
   /**
    * Raw execution, runs the method and then returns any thrown errors as the result.
    *
    * This method should never throw under any circumstances.
    */
-  async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
-    const suite = SuiteRegistryIndex.getConfig(test.class);
-
+  async #executeTestMethod(instance: unknown, test: TestConfig): Promise<Error | undefined> {
     // Ensure all the criteria below are satisfied before moving forward
     return Barrier.awaitOperation(test.timeout || TEST_TIMEOUT, async () => {
       const env = process.env;
       process.env = { ...env }; // Created an isolated environment
       try {
-        await castTo<Record<string, Function>>(suite.instance)[test.methodName]();
+        await castTo<Record<string, Function>>(instance)[test.methodName]();
       } finally {
         process.env = env; // Restore
       }
@@ -73,96 +52,44 @@ export class TestExecutor {
     }
   }
 
-  #skipTest(test: TestConfig, result: SuiteResult): void {
-    // Mark test start
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    result.skipped += 1;
-    result.total += 1;
-    this.#consumer.onEvent({
-      type: 'test',
-      phase: 'after',
-      test: {
-        ...test,
-        suiteLineStart: result.lineStart,
-        assertions: [], duration: 0, durationTotal: 0, output: [], status: 'skipped'
-      }
-    });
-  }
-
-  /**
-   * An empty suite result based on a suite config
-   */
-  createSuiteResult(suite: SuiteConfig, override?: Partial<SuiteResult>): SuiteResult {
-    return {
-      passed: 0,
-      failed: 0,
-      errored: 0,
-      skipped: 0,
-      unknown: 0,
-      total: 0,
-      status: 'unknown',
-      lineStart: suite.lineStart,
-      lineEnd: suite.lineEnd,
-      import: suite.import,
-      classId: suite.classId,
-      sourceHash: suite.sourceHash,
-      duration: 0,
-      tests: {},
-      ...override
-    };
-  }
-
   /**
    * Execute the test, capture output, assertions and promises
    */
-  async executeTest(test: TestConfig, suite: SuiteConfig): Promise<TestResult> {
+  async executeTest(instance: unknown, test: TestConfig, suite: SuiteConfig, override?: Partial<TestResult>): Promise<TestResult> {
+    const result = TestModelUtil.createTestResult(suite, test, override);
 
     // Mark test start
     this.#consumer.onEvent({ type: 'test', phase: 'before', test });
 
-    const startTime = Date.now();
-
-    const result: TestResult = {
-      methodName: test.methodName,
-      description: test.description,
-      classId: test.classId,
-      tags: test.tags,
-      suiteLineStart: suite.lineStart,
-      lineStart: test.lineStart,
-      lineEnd: test.lineEnd,
-      lineBodyStart: test.lineBodyStart,
-      import: test.import,
-      declarationImport: test.declarationImport,
-      sourceHash: test.sourceHash,
-      status: 'unknown',
-      assertions: [],
-      duration: 0,
-      durationTotal: 0,
-      output: [],
-    };
-
     // Emit every assertion as it occurs
-    const getAssertions = AssertCapture.collector(test, asrt =>
-      this.#consumer.onEvent({
-        type: 'assertion',
-        phase: 'after',
-        assertion: asrt
-      })
+    const getAssertions = AssertCapture.collector(test, item =>
+      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: item })
     );
 
     const consoleCapture = new ConsoleCapture().start(); // Capture all output from transpiled code
 
-    // Run method and get result
-    const error = await this.#executeTestMethod(test);
-    const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+    // Already finished
+    if (result.status !== 'unknown') {
+      if (result.error) {
+        result.assertions.push(AssertUtil.generateAssertion({ suite, test, error: result.error }));
+      }
+      for (const item of result.assertions ?? []) {
+        AssertCapture.add(item);
+      }
+    } else {
+      // Run method and get result
+      const startTime = Date.now();
+      const error = await this.#executeTestMethod(instance, test);
+      const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+      result.status = status;
+      result.selfDuration = Date.now() - startTime;
+      if (finalError) {
+        result.error = finalError;
+      }
+    }
 
-    Object.assign(result, {
-      status,
-      output: consoleCapture.end(),
-      assertions: getAssertions(),
-      duration: Date.now() - startTime,
-      ...(finalError ? { error: finalError } : {})
-    });
+    result.output = consoleCapture.end();
+    result.assertions = getAssertions();
 
     // Mark completion
     this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
@@ -174,19 +101,22 @@ export class TestExecutor {
    * Execute an entire suite
    */
   async executeSuite(suite: SuiteConfig, tests: TestConfig[]): Promise<void> {
+    const instance = classConstruct(suite.class);
 
-    suite.instance = classConstruct(suite.class);
+    const shouldSkip = await this.#shouldSkip(suite, instance);
 
-    const shouldSkip = await this.#shouldSkip(suite, suite.instance);
+    const result: SuiteResult = TestModelUtil.createSuiteResult(suite);
 
     if (shouldSkip) {
       this.#consumer.onEvent({
-        phase: 'after', type: 'suite',
-        suite: this.createSuiteResult(suite, {
+        phase: 'after',
+        type: 'suite',
+        suite: {
+          ...result,
           status: 'skipped',
           skipped: tests.length,
           total: tests.length
-        })
+        }
       });
     }
 
@@ -194,69 +124,82 @@ export class TestExecutor {
       return;
     }
 
-    const result: SuiteResult = this.createSuiteResult(suite);
-    const validTestMethodNames = new Set(tests.map(t => t.methodName));
-    const testConfigs = Object.fromEntries(
-      Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key))
-    );
-
+    const manager = new TestPhaseManager(suite, instance);
+    const originalEnv = { ...process.env };
     const startTime = Date.now();
+    const testResultOverrides: Record<string, Partial<TestResult>> = {};
+
+    const validTestMethodNames = new Set(tests.map(t => t.methodName));
+    const testConfigs = Object.fromEntries(Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key)));
 
     // Mark suite start
     this.#consumer.onEvent({ phase: 'before', type: 'suite', suite: { ...suite, tests: testConfigs } });
 
-    const manager = new TestPhaseManager(suite);
-
-    const originalEnv = { ...process.env };
-
     try {
       // Handle the BeforeAll calls
       await manager.startPhase('all');
+    } catch (someError) {
+      const suiteError = await manager.onError('all', someError);
+      for (const method of validTestMethodNames) {
+        testResultOverrides[method] ??= { status: 'errored', error: suiteError };
+      }
+    }
 
-      const suiteEnv = { ...process.env };
+    const suiteEnv = { ...process.env };
 
-      for (const test of tests ?? suite.tests) {
-        if (await this.#shouldSkip(test, suite.instance)) {
-          this.#skipTest(test, result);
-          continue;
-        }
+    for (const test of tests) {
+      // Reset env before each test
+      process.env = { ...suiteEnv };
 
-        // Reset env before each test
-        process.env = { ...suiteEnv };
+      const testStart = Date.now();
+      const testResultOverride = (testResultOverrides[test.methodName] ??= {});
 
-        const testStart = Date.now();
-        try {
+      if (await this.#shouldSkip(test, instance)) {
+        testResultOverride.status = 'skipped';
+      }
 
-          // Handle BeforeEach
-          await manager.startPhase('each');
+      try {
+        // Handle BeforeEach
+        testResultOverride.status || (await manager.startPhase('each'));
+      } catch (someError) {
+        const testError = await manager.onError('each', someError);
+        testResultOverride.error = testError;
+        testResultOverride.status = 'errored';
+      }
 
-          // Run test
-          const testResult = await this.executeTest(test, suite);
-          result.tests[testResult.methodName] = testResult;
-          result[testResult.status]++;
-          result.total += 1;
+      // Run test
+      const testResult = await this.executeTest(instance, test, suite, testResultOverride);
 
-          // Handle after each
-          await manager.endPhase('each');
-          testResult.durationTotal = Date.now() - testStart;
-        } catch (testError) {
-          const errors = await manager.errorPhase('each', testError, suite, test);
-          this.#recordSuiteErrors(suite, result, errors);
+      // Handle after each
+      try {
+        testResultOverride.status || (await manager.endPhase('each'));
+      } catch (testError) {
+        if (!(testError instanceof Error)) {
+          throw testError;
         }
+        console.error('Failed to properly shutdown test', testError.message);
       }
 
+      result.tests[testResult.methodName] = testResult;
+      testResult.duration = Date.now() - testStart;
+      TestModelUtil.countTestResult(result, [testResult]);
+    }
+
+    try {
       // Handle after all
       await manager.endPhase('all');
     } catch (suiteError) {
-      const errors = await manager.errorPhase('all', suiteError, suite);
-      this.#recordSuiteErrors(suite, result, errors);
+      if (!(suiteError instanceof Error)) {
+        throw suiteError;
+      }
+      console.error('Failed to properly shutdown test', suiteError.message);
     }
 
     // Restore env
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-    result.status = TestModelUtil.countsToTestStatus(result);
+    result.status = TestModelUtil.computeTestStatus(result);
 
     // Mark suite complete
     this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
@@ -265,19 +208,15 @@ export class TestExecutor {
   /**
    * Handle executing a suite's test/tests based on command line inputs
    */
-  async execute(run: TestRun): Promise<void> {
+  async execute(run: TestRun, singleFile?: boolean): Promise<void> {
     try {
       await Runtime.importFrom(run.import);
     } catch (error) {
       if (!(error instanceof Error)) {
         throw error;
       }
+      const suite = TestModelUtil.createImportErrorSuiteResult(run);
       console.error(error);
-
-      // Fire import failure as a test failure for each test in the suite
-      const { result, test, suite } = AssertUtil.gernerateImportFailure(run.import, error);
-      this.#consumer.onEvent({ type: 'suite', phase: 'before', suite });
-      this.#onSuiteTestError(result, test);
       this.#consumer.onEvent({ type: 'suite', phase: 'after', suite });
       return;
     }
@@ -291,8 +230,13 @@ export class TestExecutor {
       console.warn('Unable to find suites for ', run);
     }
 
+    if (singleFile) {
+      const testCount = suites.reduce((acc, suite) => acc + suite.tests.length, 0);
+      this.#consumer.onTestRunState?.({ testCount });
+    }
+
     for (const { suite, tests } of suites) {
       await this.executeSuite(suite, tests);
     }
   }
-}
+}

package/src/execute/phase.ts

@@ -1,9 +1,7 @@
-import { describeFunction, Env, TimeUtil } from '@travetto/runtime';
+import { Env, TimeUtil } from '@travetto/runtime';
 
 import type { SuiteConfig, SuitePhase } from '../model/suite.ts';
-import { AssertUtil } from '../assert/util.ts';
 import { Barrier } from './barrier.ts';
-import type { TestConfig, TestResult } from '../model/test.ts';
 
 const TEST_PHASE_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_PHASE_TIMEOUT.value ?? 15000, 'ms');
 
@@ -15,9 +13,11 @@ const TEST_PHASE_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_PHASE_TIMEOUT.value ??
 export class TestPhaseManager {
   #progress: ('all' | 'each')[] = [];
   #suite: SuiteConfig;
+  #instance: unknown;
 
-  constructor(suite: SuiteConfig) {
+  constructor(suite: SuiteConfig, instance: unknown) {
     this.#suite = suite;
+    this.#instance = instance;
   }
 
   /**
@@ -31,12 +31,10 @@ export class TestPhaseManager {
       }
 
       // Ensure all the criteria below are satisfied before moving forward
-      error = await Barrier.awaitOperation(TEST_PHASE_TIMEOUT, async () => handler[phase]?.(this.#suite.instance));
+      error = await Barrier.awaitOperation(TEST_PHASE_TIMEOUT, async () => handler[phase]?.(this.#instance));
 
       if (error) {
-        const toThrow = new Error(phase, { cause: error });
-        Object.assign(toThrow, { import: describeFunction(handler.constructor) ?? undefined });
-        throw toThrow;
+        throw error;
       }
     }
   }
@@ -58,21 +56,13 @@ export class TestPhaseManager {
   }
 
   /**
-   * Handles if an error occurs during a phase, ensuring that we attempt to end the phase and then return the appropriate test results for the failure
+   * Handle an error during phase operation
    */
-  async errorPhase(phase: 'all' | 'each', error: unknown, suite: SuiteConfig, test?: TestConfig): Promise<TestResult[]> {
-    try { await this.endPhase(phase); } catch { }
-    if (!(error instanceof Error)) { throw error; }
-
-    // Don't propagate our own errors
-    if (error.message === 'afterAll' || error.message === 'afterEach') {
-      return [];
-    }
-
-    if (test) {
-      return [AssertUtil.generateSuiteTestFailure({ suite, error, test })];
-    } else {
-      return AssertUtil.generateSuiteTestFailures(suite, error);
+  async onError(phase: 'all' | 'each', error: unknown): Promise<Error> {
+    if (!(error instanceof Error)) {
+      await this.endPhase(phase).catch(() => {});
+      throw error;
     }
+    return error;
   }
 }