@travetto/test 5.0.0-rc.8 → 5.0.0

package/src/execute/executor.ts

@@ -1,12 +1,11 @@
 import { AssertionError } from 'node:assert';
-import path from 'node:path';
 
-import { Env, TimeUtil, Runtime, RuntimeIndex } from '@travetto/runtime';
+import { Env, TimeUtil, Runtime, castTo } from '@travetto/runtime';
 import { Barrier, ExecutionError } from '@travetto/worker';
 
 import { SuiteRegistry } from '../registry/suite';
-import { TestConfig, TestResult } from '../model/test';
-import { SuiteConfig, SuiteResult } from '../model/suite';
+import { TestConfig, TestResult, TestRun } from '../model/test';
+import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite';
 import { TestConsumer } from '../consumer/types';
 import { AssertCheck } from '../assert/check';
 import { AssertCapture } from '../assert/capture';
@@ -22,12 +21,42 @@ const TEST_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_TIMEOUT.val) ?? 5000;
  */
 export class TestExecutor {
 
+  #consumer: TestConsumer;
+  #testFilter: (config: TestConfig) => boolean;
+
+  constructor(consumer: TestConsumer, testFilter?: (config: TestConfig) => boolean) {
+    this.#consumer = consumer;
+    this.#testFilter = testFilter || ((): boolean => true);
+  }
+
+  /**
+   * Handles communicating a suite-level error
+   * @param failure
+   * @param withSuite
+   */
+  #onSuiteFailure(failure: SuiteFailure, triggerSuite?: boolean): void {
+    if (triggerSuite) {
+      this.#consumer.onEvent({ type: 'suite', phase: 'before', suite: failure.suite });
+    }
+
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test: failure.test });
+    this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: failure.assert });
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: failure.testResult });
+
+    if (triggerSuite) {
+      this.#consumer.onEvent({
+        type: 'suite', phase: 'after',
+        suite: { ...castTo(failure.suite), failed: 1, passed: 0, total: 1, skipped: 0 }
+      });
+    }
+  }
+
   /**
    * Raw execution, runs the method and then returns any thrown errors as the result.
    *
    * This method should never throw under any circumstances.
    */
-  static async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
+  async #executeTestMethod(test: TestConfig): Promise<Error | undefined> {
     const suite = SuiteRegistry.get(test.class);
 
     // Ensure all the criteria below are satisfied before moving forward
@@ -39,8 +68,7 @@ export class TestExecutor {
 
         try {
           await pCap.run(() =>
-            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-            (suite.instance as Record<string, Function>)[test.methodName]()
+            castTo<Record<string, Function>>(suite.instance)[test.methodName]()
           );
         } finally {
           process.env = env; // Restore
@@ -54,35 +82,27 @@ export class TestExecutor {
   /**
    * Determining if we should skip
    */
-  static async #skip(cfg: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
-    if (cfg.skip !== undefined) {
-      if (typeof cfg.skip === 'boolean' ? cfg.skip : await cfg.skip(inst)) {
-        return true;
-      }
+  async #shouldSkip(cfg: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
+    if ('methodName' in cfg && !this.#testFilter(cfg)) {
+      return true;
+    }
+
+    if (typeof cfg.skip === 'function' ? await cfg.skip(inst) : cfg.skip) {
+      return true;
     }
   }
 
-  /**
-   * Fail an entire file, marking the whole file as failed
-   */
-  static failFile(consumer: TestConsumer, imp: string, err: Error): void {
-    const name = path.basename(imp);
-    const classId = `${RuntimeIndex.getFromImport(imp)?.id}￮${name}`;
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const suite = { class: { name }, classId, duration: 0, lineStart: 1, lineEnd: 1, import: imp, } as SuiteConfig & SuiteResult;
-    err.message = err.message.replaceAll(Runtime.mainSourcePath, '.');
-    const res = AssertUtil.generateSuiteError(suite, 'require', err);
-    consumer.onEvent({ type: 'suite', phase: 'before', suite });
-    consumer.onEvent({ type: 'test', phase: 'before', test: res.testConfig });
-    consumer.onEvent({ type: 'assertion', phase: 'after', assertion: res.assert });
-    consumer.onEvent({ type: 'test', phase: 'after', test: res.testResult });
-    consumer.onEvent({ type: 'suite', phase: 'after', suite: { ...suite, failed: 1, passed: 0, total: 1, skipped: 0 } });
+  #skipTest(test: TestConfig, result: SuiteResult): void {
+    // Mark test start
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
+    result.skipped++;
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: { ...test, assertions: [], duration: 0, durationTotal: 0, output: {}, status: 'skipped' } });
   }
 
   /**
    * An empty suite result based on a suite config
    */
-  static createSuiteResult(suite: SuiteConfig): SuiteResult {
+  createSuiteResult(suite: SuiteConfig): SuiteResult {
     return {
       passed: 0,
       failed: 0,
@@ -100,10 +120,10 @@ export class TestExecutor {
   /**
    * Execute the test, capture output, assertions and promises
    */
-  static async executeTest(consumer: TestConsumer, test: TestConfig, suite: SuiteConfig): Promise<TestResult> {
+  async executeTest(test: TestConfig): Promise<TestResult> {
 
     // Mark test start
-    consumer.onEvent({ type: 'test', phase: 'before', test });
+    this.#consumer.onEvent({ type: 'test', phase: 'before', test });
 
     const startTime = Date.now();
 
@@ -115,6 +135,7 @@ export class TestExecutor {
       lineEnd: test.lineEnd,
       lineBodyStart: test.lineBodyStart,
       import: test.import,
+      sourceImport: test.sourceImport,
       status: 'skipped',
       assertions: [],
       duration: 0,
@@ -122,16 +143,12 @@ export class TestExecutor {
       output: {},
     };
 
-    if (await this.#skip(test, suite.instance)) {
-      return result;
-    }
-
     // Emit every assertion as it occurs
-    const getAssertions = AssertCapture.collector(test, assrt =>
-      consumer.onEvent({
+    const getAssertions = AssertCapture.collector(test, asrt =>
+      this.#consumer.onEvent({
         type: 'assertion',
         phase: 'after',
-        assertion: assrt
+        assertion: asrt
       })
     );
 
@@ -163,47 +180,27 @@ export class TestExecutor {
     });
 
     // Mark completion
-    consumer.onEvent({ type: 'test', phase: 'after', test: result });
+    this.#consumer.onEvent({ type: 'test', phase: 'after', test: result });
 
     return result;
   }
 
   /**
-   * Execute a single test within a suite
+   * Execute an entire suite
    */
-  static async executeSuiteTest(consumer: TestConsumer, suite: SuiteConfig, test: TestConfig): Promise<void> {
-    const result: SuiteResult = this.createSuiteResult(suite);
-
-    const mgr = new TestPhaseManager(consumer, suite, result);
-
-    try {
-      await mgr.startPhase('all');
-      const skip = await this.#skip(test, suite.instance);
-      if (!skip) {
-        await mgr.startPhase('each');
-      }
-      await this.executeTest(consumer, test, suite);
-      if (!skip) {
-        await mgr.endPhase('each');
-      }
-      await mgr.endPhase('all');
-    } catch (err) {
-      await mgr.onError(err);
+  async executeSuite(suite: SuiteConfig, tests: TestConfig[]): Promise<void> {
+    if (!tests.length || await this.#shouldSkip(suite, suite.instance)) {
+      return;
     }
-  }
 
-  /**
-   * Execute an entire suite
-   */
-  static async executeSuite(consumer: TestConsumer, suite: SuiteConfig): Promise<SuiteResult> {
     const result: SuiteResult = this.createSuiteResult(suite);
 
     const startTime = Date.now();
 
     // Mark suite start
-    consumer.onEvent({ phase: 'before', type: 'suite', suite });
+    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite });
 
-    const mgr = new TestPhaseManager(consumer, suite, result);
+    const mgr = new TestPhaseManager(suite, result, e => this.#onSuiteFailure(e));
 
     const originalEnv = { ...process.env };
 
@@ -213,28 +210,30 @@ export class TestExecutor {
 
       const suiteEnv = { ...process.env };
 
-      for (const test of suite.tests) {
+      for (const test of tests ?? suite.tests) {
+        if (await this.#shouldSkip(test, suite.instance)) {
+          this.#skipTest(test, result);
+          continue;
+        }
+
         // Reset env before each test
         process.env = { ...suiteEnv };
+
         const testStart = Date.now();
-        const skip = await this.#skip(test, suite.instance);
-        if (!skip) {
-          // Handle BeforeEach
-          await mgr.startPhase('each');
-        }
+
+        // Handle BeforeEach
+        await mgr.startPhase('each');
 
         // Run test
-        const ret = await this.executeTest(consumer, test, suite);
+        const ret = await this.executeTest(test);
         result[ret.status]++;
-
-        if (!skip) {
-          result.tests.push(ret);
-        }
+        result.tests.push(ret);
 
         // Handle after each
         await mgr.endPhase('each');
         ret.durationTotal = Date.now() - testStart;
       }
+
       // Handle after all
       await mgr.endPhase('all');
     } catch (err) {
@@ -245,26 +244,23 @@ export class TestExecutor {
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-
-    // Mark suite complete
-    consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
-
     result.total = result.passed + result.failed;
 
-    return result;
+    // Mark suite complete
+    this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
   }
 
   /**
    * Handle executing a suite's test/tests based on command line inputs
    */
-  static async execute(consumer: TestConsumer, imp: string, ...args: string[]): Promise<void> {
+  async execute(run: TestRun): Promise<void> {
     try {
-      await Runtime.importFrom(imp);
+      await Runtime.importFrom(run.import);
     } catch (err) {
       if (!(err instanceof Error)) {
         throw err;
       }
-      this.failFile(consumer, imp, err);
+      this.#onSuiteFailure(AssertUtil.gernerateImportFailure(run.import, err));
       return;
     }
 
@@ -272,19 +268,10 @@ export class TestExecutor {
     await SuiteRegistry.init();
 
     // Convert inbound arguments to specific tests to run
-    const params = SuiteRegistry.getRunParams(imp, ...args);
+    const suites = SuiteRegistry.getSuiteTests(run);
 
-    // If running specific suites
-    if ('suites' in params) {
-      for (const suite of params.suites) {
-        if (!(await this.#skip(suite, suite.instance)) && suite.tests.length) {
-          await this.executeSuite(consumer, suite);
-        }
-      }
-    } else if (params.test) { // If running a single test
-      await this.executeSuiteTest(consumer, params.suite, params.test);
-    } else { // Running the suite
-      await this.executeSuite(consumer, params.suite);
+    for (const { suite, tests } of suites) {
+      await this.executeSuite(suite, tests);
     }
   }
 }

package/src/execute/phase.ts

@@ -1,12 +1,12 @@
 import { Barrier } from '@travetto/worker';
 import { Env, TimeUtil } from '@travetto/runtime';
 
-import { TestConsumer } from '../consumer/types';
-import { SuiteConfig, SuiteResult } from '../model/suite';
+import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite';
 import { AssertUtil } from '../assert/util';
-import { TestResult } from '../model/test';
 
-class TestBreakout extends Error { }
+class TestBreakout extends Error {
+  source?: Error;
+}
 
 const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.val) ?? 15000;
 
@@ -17,27 +17,14 @@ const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.val) ??
  */
 export class TestPhaseManager {
   #progress: ('all' | 'each')[] = [];
-  #consumer: TestConsumer;
   #suite: SuiteConfig;
   #result: SuiteResult;
+  #onSuiteFailure: (fail: SuiteFailure) => void;
 
-  constructor(consumer: TestConsumer, suite: SuiteConfig, result: SuiteResult) {
-    this.#consumer = consumer;
+  constructor(suite: SuiteConfig, result: SuiteResult, onSuiteFailure: (fail: SuiteFailure) => void) {
     this.#suite = suite;
     this.#result = result;
-  }
-
-  /**
-   * Create the appropriate events when a suite has an error
-   */
-  async triggerSuiteError(methodName: string, error: Error): Promise<TestResult> {
-    const bad = AssertUtil.generateSuiteError(this.#suite, methodName, error);
-
-    this.#consumer.onEvent({ type: 'test', phase: 'before', test: bad.testConfig });
-    this.#consumer.onEvent({ type: 'assertion', phase: 'after', assertion: bad.assert });
-    this.#consumer.onEvent({ type: 'test', phase: 'after', test: bad.testResult });
-
-    return bad.testResult;
+    this.#onSuiteFailure = onSuiteFailure;
   }
 
   /**
@@ -57,10 +44,9 @@ export class TestPhaseManager {
       }
     }
     if (error) {
-      const res = await this.triggerSuiteError(`[[${phase}]]`, error);
-      this.#result.tests.push(res);
-      this.#result.failed++;
-      throw new TestBreakout();
+      const tbo = new TestBreakout(`[[${phase}]]`);
+      tbo.source = error;
+      throw tbo;
     }
   }
 
@@ -96,10 +82,14 @@ export class TestPhaseManager {
 
     this.#progress = [];
 
-    if (!(err instanceof TestBreakout)) {
-      const res = await this.triggerSuiteError('all', err);
-      this.#result.tests.push(res);
-      this.#result.failed++;
-    }
+    const failure = AssertUtil.generateSuiteFailure(
+      this.#suite,
+      err instanceof TestBreakout ? err.message : 'all',
+      err instanceof TestBreakout ? err.source! : err
+    );
+
+    this.#onSuiteFailure(failure);
+    this.#result.tests.push(failure.testResult);
+    this.#result.failed++;
   }
 }

package/src/execute/promise.ts

@@ -1,4 +1,5 @@
 import { createHook, executionAsyncId } from 'node:async_hooks';
+import { isPromise } from 'node:util/types';
 
 import { ExecutionError } from '@travetto/worker';
 import { Util } from '@travetto/runtime';
@@ -11,9 +12,8 @@ export class PromiseCapturer {
   #id: number = 0;
 
   #init(id: number, type: string, triggerId: number, resource: unknown): void {
-    if (this.#id && type === 'PROMISE' && triggerId === this.#id) {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      this.#pending.set(id, resource as Promise<unknown>);
+    if (this.#id && type === 'PROMISE' && triggerId === this.#id && isPromise(resource)) {
+      this.#pending.set(id, resource);
     }
   }
 

package/src/execute/runner.ts

@@ -9,6 +9,7 @@ import { RunnableTestConsumer } from '../consumer/types/runnable';
 import { TestExecutor } from './executor';
 import { RunnerUtil } from './util';
 import { RunState } from './types';
+import { TestConfig, TestRun } from '../model/test';
 
 /**
  * Test Runner
@@ -24,22 +25,23 @@ export class Runner {
   /**
    * Run all files
    */
-  async runFiles(): Promise<boolean> {
+  async runFiles(globs?: string[]): Promise<boolean> {
     const consumer = await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format);
 
-    const imports = await RunnerUtil.getTestImports(this.#state.args);
+    console.debug('Running', { globs });
 
-    console.debug('Running', { patterns: this.#state.args });
+    const tests = await RunnerUtil.getTestDigest(globs, this.#state.tags);
+    const testRuns = RunnerUtil.getTestRuns(tests)
+      .sort((a, b) => a.runId!.localeCompare(b.runId!));
 
-    const testCount = await RunnerUtil.getTestCount(this.#state.args);
-    await consumer.onStart({ testCount });
+    await consumer.onStart({ testCount: tests.length });
     await WorkPool.run(
-      buildStandardTestManager.bind(null, consumer),
-      imports,
+      f => buildStandardTestManager(consumer, f),
+      testRuns,
       {
         idleTimeoutMillis: TimeUtil.asMillis(10, 's'),
         min: 1,
-        max: this.#state.concurrency,
+        max: this.#state.concurrency
       });
 
     return consumer.summarizeAsBoolean();
@@ -48,24 +50,30 @@ export class Runner {
   /**
    * Run a single file
    */
-  async runSingle(): Promise<boolean> {
-    let imp = RuntimeIndex.getFromImport(this.#state.args[0])?.import;
+  async runSingle(run: TestRun): Promise<boolean> {
+    run.import =
+      RuntimeIndex.getFromImport(run.import)?.import ??
+      RuntimeIndex.getFromSource(path.resolve(run.import))?.import!;
 
-    if (!imp) {
-      imp = RuntimeIndex.getFromSource(path.resolve(this.#state.args[0]))?.import;
-    }
+    const entry = RuntimeIndex.getFromImport(run.import)!;
 
-    const entry = RuntimeIndex.getFromImport(imp!)!;
     if (entry.module !== Runtime.main.name) {
       RuntimeIndex.reinitForModule(entry.module);
     }
 
-    const consumer = await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format);
+    const filter = (run.methodNames?.length) ?
+      (cfg: TestConfig): boolean => run.methodNames!.includes(cfg.methodName) :
+      undefined;
 
-    const [, ...args] = this.#state.args;
+    const consumer = (await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format))
+      .withTransformer(e => {
+        // Copy run metadata to event
+        e.metadata = run.metadata;
+        return e;
+      });
 
     await consumer.onStart({});
-    await TestExecutor.execute(consumer, imp!, ...args);
+    await new TestExecutor(consumer, filter).execute(run);
     return consumer.summarizeAsBoolean();
   }
 
@@ -73,9 +81,10 @@ export class Runner {
    * Run the runner, based on the inputs passed to the constructor
    */
   async run(): Promise<boolean | undefined> {
-    switch (this.#state.mode) {
-      case 'single': return await this.runSingle();
-      case 'standard': return await this.runFiles();
+    if ('import' in this.#state.target) {
+      return await this.runSingle(this.#state.target);
+    } else {
+      return await this.runFiles(this.#state.target.globs);
     }
   }
 }

package/src/execute/types.ts

@@ -1,4 +1,5 @@
 import { TestConsumer } from '../consumer/types';
+import { TestRun } from '../model/test';
 
 /**
  * Run state
@@ -13,19 +14,20 @@ export interface RunState {
    */
   consumer?: TestConsumer;
   /**
-   * Test mode
-   */
-  mode?: 'single' | 'watch' | 'standard';
-  /**
-   * Show progress to stderr
+   * Number of test suites to run concurrently, when mode is not single
    */
-  showProgress?: boolean;
+  concurrency?: number;
   /**
-   * Number of test suites to run concurrently, when mode is not single
+   * The tags to include or exclude from testing
    */
-  concurrency: number;
+  tags?: string[];
   /**
-   * Input arguments
+   * target
    */
-  args: string[];
+  target: TestRun | {
+    /**
+     * Globs to run
+     */
+    globs?: string[];
+  };
 }

package/src/execute/util.ts

@@ -4,6 +4,7 @@ import fs from 'node:fs/promises';
 import readline from 'node:readline/promises';
 
 import { Env, ExecUtil, ShutdownManager, Util, RuntimeIndex, Runtime } from '@travetto/runtime';
+import { TestConfig, TestRun } from '../model/test';
 
 /**
  * Simple Test Utilities
@@ -63,13 +64,13 @@ export class RunnerUtil {
   }
 
   /**
-   * Get count of tests for a given set of patterns
-   * @param patterns
+   * Get count of tests for a given set of globs
+   * @param globs
    * @returns
    */
-  static async getTestCount(patterns: string[]): Promise<number> {
+  static async getTestDigest(globs: string[] = ['**/*.ts'], tags?: string[]): Promise<TestConfig[]> {
     const countRes = await ExecUtil.getResult(
-      spawn('npx', ['trv', 'test:count', ...patterns], {
+      spawn('npx', ['trv', 'test:digest', '-o', 'json', ...globs], {
         env: { ...process.env, ...Env.FORCE_COLOR.export(0), ...Env.NO_COLOR.export(true) }
       }),
       { catch: true }
@@ -77,6 +78,30 @@ export class RunnerUtil {
     if (!countRes.valid) {
       throw new Error(countRes.stderr);
     }
-    return countRes.valid ? +countRes.stdout : 0;
+
+    const testFilter = tags?.length ?
+      Util.allowDeny<string, [TestConfig]>(
+        tags,
+        rule => rule,
+        (rule, core) => core.tags?.includes(rule) ?? false
+      ) :
+      ((): boolean => true);
+
+    const res: TestConfig[] = countRes.valid ? JSON.parse(countRes.stdout) : [];
+    return res.filter(testFilter);
+  }
+
+  /**
+   * Get run events
+   */
+  static getTestRuns(tests: TestConfig[]): TestRun[] {
+    const events = tests.reduce((acc, test) => {
+      if (!acc.has(test.classId)) {
+        acc.set(test.classId, { import: test.import, classId: test.classId, methodNames: [], runId: Util.uuid() });
+      }
+      acc.get(test.classId)!.methodNames!.push(test.methodName);
+      return acc;
+    }, new Map<string, TestRun>());
+    return [...events.values()];
   }
 }