@travetto/test 8.0.0-alpha.0 → 8.0.0-alpha.10

package/src/execute/executor.ts

@@ -26,38 +26,18 @@ export class TestExecutor {
     this.#consumer = consumer;
   }
 
-  #onSuiteTestError(result: TestResult, test: TestConfig): void {
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    for (const assertion of result.assertions) {
-      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion });
-    }
-    this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
-  }
-
-  #recordSuiteErrors(suiteConfig: SuiteConfig, suiteResult: SuiteResult, errors: TestResult[]): void {
-    for (const test of errors) {
-      if (!suiteResult.tests[test.methodName]) {
-        this.#onSuiteTestError(test, suiteConfig.tests[test.methodName]);
-        suiteResult.errored += 1;
-        suiteResult.total += 1;
-      }
-    }
-  }
-
   /**
    * Raw execution, runs the method and then returns any thrown errors as the result.
    *
    * This method should never throw under any circumstances.
    */
-  async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
-    const suite = SuiteRegistryIndex.getConfig(test.class);
-
+  async #executeTestMethod(instance: unknown, test: TestConfig): Promise<Error | undefined> {
     // Ensure all the criteria below are satisfied before moving forward
     return Barrier.awaitOperation(test.timeout || TEST_TIMEOUT, async () => {
       const env = process.env;
       process.env = { ...env }; // Created an isolated environment
       try {
-        await castTo<Record<string, Function>>(suite.instance)[test.methodName]();
+        await castTo<Record<string, Function>>(instance)[test.methodName]();
       } finally {
         process.env = env; // Restore
       }
@@ -73,95 +53,43 @@ export class TestExecutor {
     }
   }
 
-  #skipTest(test: TestConfig, result: SuiteResult): void {
-    // Mark test start
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
-    result.skipped += 1;
-    result.total += 1;
-    this.#consumer.onEvent({
-      type: 'test',
-      phase: 'after',
-      test: {
-        ...test,
-        suiteLineStart: result.lineStart,
-        assertions: [], duration: 0, durationTotal: 0, output: [], status: 'skipped'
-      }
-    });
-  }
-
-  /**
-   * An empty suite result based on a suite config
-   */
-  createSuiteResult(suite: SuiteConfig): SuiteResult {
-    return {
-      passed: 0,
-      failed: 0,
-      errored: 0,
-      skipped: 0,
-      unknown: 0,
-      total: 0,
-      status: 'unknown',
-      lineStart: suite.lineStart,
-      lineEnd: suite.lineEnd,
-      import: suite.import,
-      classId: suite.classId,
-      sourceHash: suite.sourceHash,
-      duration: 0,
-      tests: {}
-    };
-  }
-
   /**
    * Execute the test, capture output, assertions and promises
    */
-  async executeTest(test: TestConfig, suite: SuiteConfig): Promise<TestResult> {
+  async executeTest(instance: unknown, test: TestConfig, suite: SuiteConfig, override?: Partial<TestResult>): Promise<TestResult> {
+
+    const result = TestModelUtil.createTestResult(suite, test, override);
 
     // Mark test start
     this.#consumer.onEvent({ type: 'test', phase: 'before', test });
 
-    const startTime = Date.now();
-
-    const result: TestResult = {
-      methodName: test.methodName,
-      description: test.description,
-      classId: test.classId,
-      tags: test.tags,
-      suiteLineStart: suite.lineStart,
-      lineStart: test.lineStart,
-      lineEnd: test.lineEnd,
-      lineBodyStart: test.lineBodyStart,
-      import: test.import,
-      declarationImport: test.declarationImport,
-      sourceHash: test.sourceHash,
-      status: 'unknown',
-      assertions: [],
-      duration: 0,
-      durationTotal: 0,
-      output: [],
-    };
 
     // Emit every assertion as it occurs
-    const getAssertions = AssertCapture.collector(test, asrt =>
-      this.#consumer.onEvent({
-        type: 'assertion',
-        phase: 'after',
-        assertion: asrt
-      })
-    );
+    const getAssertions = AssertCapture.collector(test, item =>
+      this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: item }));
 
     const consoleCapture = new ConsoleCapture().start(); // Capture all output from transpiled code
 
-    // Run method and get result
-    const error = await this.#executeTestMethod(test);
-    const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+    // Already finished
+    if (result.status !== 'unknown') {
+      if (result.error) {
+        result.assertions.push(AssertUtil.generateAssertion({ suite, test, error: result.error }));
+      }
+      for (const item of result.assertions ?? []) { AssertCapture.add(item); }
+    } else {
+      // Run method and get result
+      const startTime = Date.now();
+      const error = await this.#executeTestMethod(instance, test);
+      const [status, finalError] = AssertCheck.validateTestResultError(test, error);
+      result.status = status;
+      result.selfDuration = Date.now() - startTime;
+      if (finalError) {
+        result.error = finalError;
+      }
+    }
 
-    Object.assign(result, {
-      status,
-      output: consoleCapture.end(),
-      assertions: getAssertions(),
-      duration: Date.now() - startTime,
-      ...(finalError ? { error: finalError } : {})
-    });
+    result.output = consoleCapture.end();
+    result.assertions = getAssertions();
 
     // Mark completion
     this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
@@ -174,75 +102,102 @@ export class TestExecutor {
    */
   async executeSuite(suite: SuiteConfig, tests: TestConfig[]): Promise<void> {
 
-    suite.instance = classConstruct(suite.class);
+    const instance = classConstruct(suite.class);
 
-    if (!tests.length || await this.#shouldSkip(suite, suite.instance)) {
+    const shouldSkip = await this.#shouldSkip(suite, instance);
+
+    const result: SuiteResult = TestModelUtil.createSuiteResult(suite);
+
+    if (shouldSkip) {
+      this.#consumer.onEvent({
+        phase: 'after', type: 'suite',
+        suite: {
+          ...result,
+          status: 'skipped',
+          skipped: tests.length,
+          total: tests.length
+        }
+      });
+    }
+
+    if (shouldSkip || !tests.length) {
       return;
     }
 
-    const result: SuiteResult = this.createSuiteResult(suite);
+    const manager = new TestPhaseManager(suite, instance);
+    const originalEnv = { ...process.env };
+    const startTime = Date.now();
+    const testResultOverrides: Record<string, Partial<TestResult>> = {};
+
     const validTestMethodNames = new Set(tests.map(t => t.methodName));
     const testConfigs = Object.fromEntries(
       Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key))
     );
 
-    const startTime = Date.now();
-
     // Mark suite start
     this.#consumer.onEvent({ phase: 'before', type: 'suite', suite: { ...suite, tests: testConfigs } });
 
-    const manager = new TestPhaseManager(suite);
-
-    const originalEnv = { ...process.env };
-
     try {
       // Handle the BeforeAll calls
       await manager.startPhase('all');
+    } catch (someError) {
+      const suiteError = await manager.onError('all', someError);
+      for (const method of validTestMethodNames) {
+        testResultOverrides[method] ??= { status: 'errored', error: suiteError };
+      }
+    }
 
-      const suiteEnv = { ...process.env };
+    const suiteEnv = { ...process.env };
 
-      for (const test of tests ?? suite.tests) {
-        if (await this.#shouldSkip(test, suite.instance)) {
-          this.#skipTest(test, result);
-          continue;
-        }
+    for (const test of tests) {
+      // Reset env before each test
+      process.env = { ...suiteEnv };
 
-        // Reset env before each test
-        process.env = { ...suiteEnv };
+      const testStart = Date.now();
+      const testResultOverride = (testResultOverrides[test.methodName] ??= {});
 
-        const testStart = Date.now();
-        try {
+      if (await this.#shouldSkip(test, instance)) {
+        testResultOverride.status = 'skipped';
+      }
 
-          // Handle BeforeEach
-          await manager.startPhase('each');
+      try {
+        // Handle BeforeEach
+        testResultOverride.status || await manager.startPhase('each');
+      } catch (someError) {
+        const testError = await manager.onError('each', someError);
+        testResultOverride.error = testError;
+        testResultOverride.status = 'errored';
+      }
 
-          // Run test
-          const testResult = await this.executeTest(test, suite);
-          result.tests[testResult.methodName] = testResult;
-          result[testResult.status]++;
-          result.total += 1;
+      // Run test
+      const testResult = await this.executeTest(instance, test, suite, testResultOverride);
 
-          // Handle after each
-          await manager.endPhase('each');
-          testResult.durationTotal = Date.now() - testStart;
-        } catch (testError) {
-          const errors = await manager.errorPhase('each', testError, suite, test);
-          this.#recordSuiteErrors(suite, result, errors);
-        }
+      // Handle after each
+      try {
+        testResultOverride.status || await manager.endPhase('each');
+      } catch (testError) {
+        if (!(testError instanceof Error)) { throw testError; };
+        console.error('Failed to properly shutdown test', testError.message);
       }
 
+      result.tests[testResult.methodName] = testResult;
+      testResult.duration = Date.now() - testStart;
+      TestModelUtil.countTestResult(result, [testResult]);
+    }
+
+    try {
       // Handle after all
       await manager.endPhase('all');
     } catch (suiteError) {
-      const errors = await manager.errorPhase('all', suiteError, suite);
-      this.#recordSuiteErrors(suite, result, errors);
+      if (!(suiteError instanceof Error)) { throw suiteError; };
+      console.error('Failed to properly shutdown test', suiteError.message);
     }
 
     // Restore env
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-    result.status = TestModelUtil.countsToTestStatus(result);
+    result.status = TestModelUtil.computeTestStatus(result);
 
     // Mark suite complete
     this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
@@ -251,19 +206,13 @@ export class TestExecutor {
   /**
    * Handle executing a suite's test/tests based on command line inputs
    */
-  async execute(run: TestRun): Promise<void> {
+  async execute(run: TestRun, singleFile?: boolean): Promise<void> {
     try {
       await Runtime.importFrom(run.import);
     } catch (error) {
-      if (!(error instanceof Error)) {
-        throw error;
-      }
+      if (!(error instanceof Error)) { throw error; }
+      const suite = TestModelUtil.createImportErrorSuiteResult(run);
       console.error(error);
-
-      // Fire import failure as a test failure for each test in the suite
-      const { result, test, suite } = AssertUtil.gernerateImportFailure(run.import, error);
-      this.#consumer.onEvent({ type: 'suite', phase: 'before', suite });
-      this.#onSuiteTestError(result, test);
       this.#consumer.onEvent({ type: 'suite', phase: 'after', suite });
       return;
     }
@@ -277,6 +226,11 @@ export class TestExecutor {
       console.warn('Unable to find suites for ', run);
     }
 
+    if (singleFile) {
+      const testCount = suites.reduce((acc, suite) => acc + suite.tests.length, 0);
+      this.#consumer.onTestRunState?.({ testCount });
+    }
+
     for (const { suite, tests } of suites) {
       await this.executeSuite(suite, tests);
     }

package/src/execute/phase.ts

@@ -1,9 +1,7 @@
-import { describeFunction, Env, TimeUtil } from '@travetto/runtime';
+import { Env, TimeUtil } from '@travetto/runtime';
 
 import type { SuiteConfig, SuitePhase } from '../model/suite.ts';
-import { AssertUtil } from '../assert/util.ts';
 import { Barrier } from './barrier.ts';
-import type { TestConfig, TestResult } from '../model/test.ts';
 
 const TEST_PHASE_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_PHASE_TIMEOUT.value ?? 15000, 'ms');
 
@@ -15,9 +13,11 @@ const TEST_PHASE_TIMEOUT = TimeUtil.duration(Env.TRV_TEST_PHASE_TIMEOUT.value ??
 export class TestPhaseManager {
   #progress: ('all' | 'each')[] = [];
   #suite: SuiteConfig;
+  #instance: unknown;
 
-  constructor(suite: SuiteConfig) {
+  constructor(suite: SuiteConfig, instance: unknown) {
     this.#suite = suite;
+    this.#instance = instance;
   }
 
   /**
@@ -31,12 +31,10 @@ export class TestPhaseManager {
       }
 
       // Ensure all the criteria below are satisfied before moving forward
-      error = await Barrier.awaitOperation(TEST_PHASE_TIMEOUT, async () => handler[phase]?.(this.#suite.instance));
+      error = await Barrier.awaitOperation(TEST_PHASE_TIMEOUT, async () => handler[phase]?.(this.#instance));
 
       if (error) {
-        const toThrow = new Error(phase, { cause: error });
-        Object.assign(toThrow, { import: describeFunction(handler.constructor) ?? undefined });
-        throw toThrow;
+        throw error;
       }
     }
   }
@@ -58,21 +56,13 @@ export class TestPhaseManager {
   }
 
   /**
-   * Handles if an error occurs during a phase, ensuring that we attempt to end the phase and then return the appropriate test results for the failure
+   * Handle an error during phase operation
    */
-  async errorPhase(phase: 'all' | 'each', error: unknown, suite: SuiteConfig, test?: TestConfig): Promise<TestResult[]> {
-    try { await this.endPhase(phase); } catch { }
-    if (!(error instanceof Error)) { throw error; }
-
-    // Don't propagate our own errors
-    if (error.message === 'afterAll' || error.message === 'afterEach') {
-      return [];
-    }
-
-    if (test) {
-      return [AssertUtil.generateSuiteTestFailure({ suite, error, test })];
-    } else {
-      return AssertUtil.generateSuiteTestFailures(suite, error);
+  async onError(phase: 'all' | 'each', error: unknown): Promise<Error> {
+    if (!(error instanceof Error)) {
+      await this.endPhase(phase).catch(() => { });
+      throw error;
     }
+    return error;
   }
 }

package/src/execute/run.ts

@@ -3,7 +3,7 @@ import fs from 'node:fs/promises';
 import readline from 'node:readline/promises';
 import path from 'node:path';
 
-import { Env, ExecUtil, Util, RuntimeIndex, Runtime, TimeUtil, JSONUtil } from '@travetto/runtime';
+import { Env, ExecUtil, Util, RuntimeIndex, Runtime, TimeUtil, JSONUtil, describeFunction } from '@travetto/runtime';
 import { WorkPool } from '@travetto/worker';
 import { Registry } from '@travetto/registry';
 
@@ -126,6 +126,7 @@ export class RunUtil {
     const imported = await Registry.manualInit([importPath]);
     const classes = Object.fromEntries(
       imported
+        .filter(cls => !describeFunction(cls).abstract)
         .filter(cls => SuiteRegistryIndex.hasConfig(cls))
         .map(cls => [cls.Ⲑid, SuiteRegistryIndex.getConfig(cls)])
     );
@@ -222,7 +223,7 @@ export class RunUtil {
     }
 
     if (runs.length === 1) {
-      await new TestExecutor(consumer).execute(runs[0]);
+      await new TestExecutor(consumer).execute(runs[0], true);
     } else {
       await WorkPool.run(
         run => buildStandardTestManager(consumer, run),

package/src/model/suite.ts

@@ -19,10 +19,6 @@ export interface SuiteConfig extends SuiteCore {
    * Should this be skipped
    */
   skip: Skip;
-  /**
-   * Actual class instance
-   */
-  instance?: unknown;
   /**
    * Tests to run
    */
@@ -34,29 +30,35 @@ export interface SuiteConfig extends SuiteCore {
 }
 
 /**
- * All counts for the suite summary
+ * Test Counts
  */
-export interface Counts {
+export interface ResultsSummary {
+  /** Passing Test Count  */
   passed: number;
+  /** Skipped Test Count  */
   skipped: number;
+  /** Failed Test Count  */
   failed: number;
+  /** Errored Test Count  */
   errored: number;
+  /** Unknown Test Count  */
   unknown: number;
+  /** Total Test Count  */
   total: number;
+  /** Test Self Execution Duration  */
+  selfDuration: number;
+  /** Total Duration */
+  duration: number;
 }
 
 /**
  * Results of a suite run
  */
-export interface SuiteResult extends Counts, SuiteCore {
+export interface SuiteResult extends ResultsSummary, SuiteCore {
   /**
    * All test results
    */
   tests: Record<string, TestResult>;
-  /**
-   * Suite duration
-   */
-  duration: number;
   /**
    * Overall status
    */

package/src/model/test.ts

@@ -105,13 +105,13 @@ export interface TestResult extends TestCore {
    */
   assertions: Assertion[];
   /**
-   * Duration for the test
+   * Self Execution Duration
    */
-  duration: number;
+  selfDuration: number;
   /**
    * Total duration including before/after
    */
-  durationTotal: number;
+  duration: number;
   /**
    * Logging output
    */

package/src/model/util.ts

@@ -1,14 +1,97 @@
-import type { Counts } from './suite.ts';
-import type { TestStatus } from './test.ts';
+import path from 'node:path';
+
+import { asFull, RuntimeIndex } from '@travetto/runtime';
+
+import type { ResultsSummary, SuiteConfig, SuiteResult } from './suite.ts';
+import type { TestConfig, TestResult, TestRun, TestStatus } from './test.ts';
 
 export class TestModelUtil {
-  static countsToTestStatus(counts: Counts): TestStatus {
+  static computeTestStatus(summary: ResultsSummary): TestStatus {
     switch (true) {
-      case counts.errored > 0: return 'errored';
-      case counts.failed > 0: return 'failed';
-      case counts.skipped > 0: return 'skipped';
-      case counts.unknown > 0: return 'unknown';
+      case summary.errored > 0: return 'errored';
+      case summary.failed > 0: return 'failed';
+      case summary.skipped > 0: return 'skipped';
+      case summary.unknown > 0: return 'unknown';
       default: return 'passed';
     }
   }
+
+  static buildSummary(): ResultsSummary {
+    return { passed: 0, failed: 0, skipped: 0, errored: 0, unknown: 0, total: 0, duration: 0, selfDuration: 0 };
+  }
+
+  static countTestResult<T extends ResultsSummary>(summary: T, tests: Pick<TestResult, 'status' | 'selfDuration' | 'duration'>[]): T {
+    for (const test of tests) {
+      summary[test.status] += 1;
+      summary.total += 1;
+      summary.selfDuration += (test.selfDuration ?? 0);
+      summary.duration += (test.duration ?? 0);
+    }
+    return summary;
+  }
+
+
+  /**
+   * An empty suite result based on a suite config
+   */
+  static createSuiteResult(suite: SuiteConfig, override?: Partial<SuiteResult>): SuiteResult {
+    return {
+      ...TestModelUtil.buildSummary(),
+      status: 'unknown',
+      lineStart: suite.lineStart,
+      lineEnd: suite.lineEnd,
+      import: suite.import,
+      classId: suite.classId,
+      sourceHash: suite.sourceHash,
+      tests: {},
+      duration: 0,
+      selfDuration: 0,
+      ...override
+    };
+  }
+
+  /**
+   * An empty test result based on a suite and test config
+   */
+  static createTestResult(suite: SuiteConfig, test: TestConfig, override?: Partial<TestResult>): TestResult {
+    return {
+      methodName: test.methodName,
+      description: test.description,
+      classId: test.classId,
+      tags: test.tags,
+      suiteLineStart: suite.lineStart,
+      lineStart: test.lineStart,
+      lineEnd: test.lineEnd,
+      lineBodyStart: test.lineBodyStart,
+      import: test.import,
+      declarationImport: test.declarationImport,
+      sourceHash: test.sourceHash,
+      status: 'unknown',
+      assertions: [],
+      duration: 0,
+      selfDuration: 0,
+      output: [],
+      ...override
+    };
+  }
+
+  static createImportErrorSuiteResult(run: TestRun): SuiteResult {
+    const name = path.basename(run.import);
+    const classId = `${RuntimeIndex.getFromImport(run.import)?.id}#${name}`;
+    const common = { classId, duration: 0, lineStart: 1, lineEnd: 1, import: run.import } as const;
+    return asFull<SuiteResult>({
+      ...common,
+      status: 'errored', errored: 1,
+      tests: {
+        impport: asFull<TestResult>({
+          ...common,
+          status: 'errored',
+          assertions: [{
+            ...common, line: common.lineStart,
+            methodName: 'import', operator: 'import', text: `Failed to import ${run.import}`,
+          }]
+        })
+      }
+    });
+  }
 }

package/src/registry/registry-adapter.ts

@@ -8,7 +8,7 @@ import type { TestConfig } from '../model/test.ts';
 function combineClasses(baseConfig: SuiteConfig, ...subConfig: Partial<SuiteConfig>[]): SuiteConfig {
   for (const config of subConfig) {
     if (config.tags) {
-      baseConfig.tags = [...baseConfig.tags ?? [], ...config.tags];
+      baseConfig.tags = [...new Set([...baseConfig.tags ?? [], ...config.tags])];
     }
     baseConfig.skip = config.skip ?? baseConfig.skip;
 
@@ -30,6 +30,21 @@ function combineClasses(baseConfig: SuiteConfig, ...subConfig: Partial<SuiteConf
   return baseConfig;
 }
 
+function combineWithParent(baseConfig: SuiteConfig, parentConfig: SuiteConfig): SuiteConfig {
+  baseConfig.tags = [...parentConfig.tags ?? [], ...baseConfig.tags ?? []];
+  baseConfig.skip = baseConfig.skip ?? parentConfig.skip;
+  baseConfig.phaseHandlers = [...(parentConfig.phaseHandlers ?? []), ...(baseConfig.phaseHandlers ?? [])];
+  for (const [key, test] of Object.entries(parentConfig.tests ?? {})) {
+    baseConfig.tests[key] = {
+      ...test,
+      class: baseConfig.class,
+      classId: baseConfig.classId,
+      import: baseConfig.import,
+    };
+  }
+  return baseConfig;
+}
+
 function combineMethods(suite: SuiteConfig, baseConfig: TestConfig, ...subConfig: Partial<TestConfig>[]): TestConfig {
   baseConfig.classId = suite.classId;
   baseConfig.import = suite.import;
@@ -54,12 +69,12 @@ export class SuiteRegistryAdapter implements RegistryAdapter<SuiteConfig> {
 
   register(...data: Partial<SuiteConfig>[]): SuiteConfig {
     if (!this.#config) {
-      const { lines, hash } = describeFunction(this.#cls) ?? {};
+      const { lines, hash, abstract: isAbstract } = describeFunction(this.#cls) ?? {};
       this.#config = asFull<SuiteConfig>({
         class: this.#cls,
         classId: this.#cls.Ⲑid,
         tags: [],
-        skip: false,
+        skip: isAbstract,
         import: Runtime.getImport(this.#cls),
         lineStart: lines?.[0],
         lineEnd: lines?.[1],
@@ -99,11 +114,11 @@ export class SuiteRegistryAdapter implements RegistryAdapter<SuiteConfig> {
 
   finalize(parent?: SuiteConfig): void {
     if (parent) {
-      combineClasses(this.#config, parent);
+      combineWithParent(this.#config, parent);
     }
 
     for (const test of Object.values(this.#config.tests)) {
-      test.tags = [...test.tags ?? [], ...this.#config.tags ?? []];
+      test.tags = [...new Set([...test.tags ?? [], ...this.#config.tags ?? []])];
       test.description ||= SchemaRegistryIndex.get(this.#cls).getMethod(test.methodName).description;
     }
   }

package/src/worker/child.ts

@@ -20,9 +20,7 @@ export class TestChildWorker extends IpcChannel<TestRun> {
       await operation();
       this.send(type); // Respond
     } catch (error) {
-      if (!(error instanceof Error)) {
-        throw error;
-      }
+      if (!(error instanceof Error)) { throw error; }
       // Mark as errored out
       this.send(type, JSONUtil.cloneForTransmit(error));
     }