@travetto/test 7.0.0-rc.1 → 7.0.0-rc.3

package/src/decorator/test.ts

@@ -3,13 +3,15 @@ import { ClassInstance, getClass } from '@travetto/runtime';
 import { TestConfig, ThrowableError } from '../model/test.ts';
 import { SuiteRegistryIndex } from '../registry/registry-index.ts';
 
+type MethodDecorator = (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => PropertyDescriptor | void;
+
 /**
  * The `@AssertCheck` indicates that a function's assert calls should be transformed
  * @augments `@travetto/test:AssertCheck`
  * @kind decorator
  */
 export function AssertCheck(): MethodDecorator {
-  return (instance: ClassInstance, property: string | symbol, descriptor: PropertyDescriptor) => descriptor;
+  return (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => descriptor;
 }
 
 /**
@@ -24,7 +26,7 @@ export function Test(): MethodDecorator;
 export function Test(...rest: Partial<TestConfig>[]): MethodDecorator;
 export function Test(description: string, ...rest: Partial<TestConfig>[]): MethodDecorator;
 export function Test(description?: string | Partial<TestConfig>, ...rest: Partial<TestConfig>[]): MethodDecorator {
-  return (instance: ClassInstance, property: string | symbol, descriptor: PropertyDescriptor) => {
+  return (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => {
     SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(property, descriptor.value,
       ...(typeof description !== 'string' && description) ? [description] : [],
       ...rest,
@@ -40,7 +42,7 @@ export function Test(description?: string | Partial<TestConfig>, ...rest: Partia
  * @kind decorator
  */
 export function ShouldThrow(state: ThrowableError): MethodDecorator {
-  return (instance: ClassInstance, property: string | symbol, descriptor: PropertyDescriptor) => {
+  return (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => {
     SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(property, descriptor.value, { shouldThrow: state });
     return descriptor;
   };
@@ -52,7 +54,7 @@ export function ShouldThrow(state: ThrowableError): MethodDecorator {
  * @kind decorator
  */
 export function Timeout(ms: number): MethodDecorator {
-  return (instance: ClassInstance, property: string | symbol, descriptor: PropertyDescriptor) => {
+  return (instance: ClassInstance, property: string, descriptor: PropertyDescriptor) => {
     SuiteRegistryIndex.getForRegister(getClass(instance)).registerTest(property, descriptor.value, { timeout: ms });
     return descriptor;
   };

package/src/execute/barrier.ts

@@ -11,14 +11,14 @@ export class Barrier {
   /**
    * Track timeout
    */
-  static timeout(duration: number | TimeSpan, op: string = 'Operation'): { promise: Promise<void>, resolve: () => unknown } {
+  static timeout(duration: number | TimeSpan, operation: string = 'Operation'): { promise: Promise<void>, resolve: () => unknown } {
     const resolver = Promise.withResolvers<void>();
     const durationMs = TimeUtil.asMillis(duration);
     let timeout: NodeJS.Timeout;
     if (!durationMs) {
       resolver.resolve();
     } else {
-      const msg = `${op} timed out after ${duration}${typeof duration === 'number' ? 'ms' : ''}`;
+      const msg = `${operation} timed out after ${duration}${typeof duration === 'number' ? 'ms' : ''}`;
       timeout = setTimeout(() => resolver.reject(new TimeoutError(msg)), durationMs).unref();
     }
 
@@ -31,9 +31,9 @@ export class Barrier {
    */
   static uncaughtErrorPromise(): { promise: Promise<void>, resolve: () => unknown } {
     const uncaught = Promise.withResolvers<void>();
-    const onError = (err: Error): void => { Util.queueMacroTask().then(() => uncaught.reject(err)); };
-    UNCAUGHT_ERR_EVENTS.map(k => process.on(k, onError));
-    uncaught.promise.finally(() => { UNCAUGHT_ERR_EVENTS.map(k => process.off(k, onError)); });
+    const onError = (error: Error): void => { Util.queueMacroTask().then(() => uncaught.reject(error)); };
+    UNCAUGHT_ERR_EVENTS.map(key => process.on(key, onError));
+    uncaught.promise.finally(() => { UNCAUGHT_ERR_EVENTS.map(key => process.off(key, onError)); });
     return uncaught;
   }
 
@@ -80,7 +80,7 @@ export class Barrier {
   /**
    * Wait for operation to finish, with timeout and unhandled error support
    */
-  static async awaitOperation(timeout: number | TimeSpan, op: () => Promise<unknown>): Promise<Error | undefined> {
+  static async awaitOperation(timeout: number | TimeSpan, operation: () => Promise<unknown>): Promise<Error | undefined> {
     const uncaught = this.uncaughtErrorPromise();
     const timer = this.timeout(timeout);
     const promises = this.capturePromises();
@@ -88,9 +88,9 @@ export class Barrier {
     try {
       await promises.start();
       let capturedError: Error | undefined;
-      const opProm = op().then(() => promises.finish());
+      const opProm = operation().then(() => promises.finish());
 
-      await Promise.race([opProm, uncaught.promise, timer.promise]).catch(err => capturedError ??= err);
+      await Promise.race([opProm, uncaught.promise, timer.promise]).catch(error => capturedError ??= error);
 
       return capturedError;
     } finally {

package/src/execute/console.ts

@@ -22,7 +22,7 @@ export class ConsoleCapture implements ConsoleListener {
     this.out.push({
       ...rest,
       message: args
-        .map((x => typeof x === 'string' ? x : util.inspect(x, false, 5)))
+        .map((arg => typeof arg === 'string' ? arg : util.inspect(arg, false, 5)))
         .join(' ')
     });
   }

package/src/execute/executor.ts

@@ -3,7 +3,7 @@ import { AssertionError } from 'node:assert';
 import { Env, TimeUtil, Runtime, castTo, classConstruct } from '@travetto/runtime';
 import { Registry } from '@travetto/registry';
 
-import { TestConfig, TestResult, TestRun } from '../model/test.ts';
+import { TestConfig, TestResult, type TestRun } from '../model/test.ts';
 import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite.ts';
 import { TestConsumerShape } from '../consumer/types.ts';
 import { AssertCheck } from '../assert/check.ts';
@@ -14,8 +14,9 @@ import { AssertUtil } from '../assert/util.ts';
 import { Barrier } from './barrier.ts';
 import { ExecutionError } from './error.ts';
 import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+import { TestModelUtil } from '../model/util.ts';
 
-const TEST_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_TIMEOUT.val) ?? 5000;
+const TEST_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_TIMEOUT.value) ?? 5000;
 
 /**
  * Support execution of the tests
@@ -73,8 +74,8 @@ export class TestExecutor {
   /**
    * Determining if we should skip
    */
-  async #shouldSkip(cfg: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
-    if (typeof cfg.skip === 'function' ? await cfg.skip(inst) : cfg.skip) {
+  async #shouldSkip(config: TestConfig | SuiteConfig, inst: unknown): Promise<boolean | undefined> {
+    if (typeof config.skip === 'function' ? await config.skip(inst) : config.skip) {
       return true;
     }
   }
@@ -94,13 +95,16 @@ export class TestExecutor {
       passed: 0,
       failed: 0,
       skipped: 0,
+      unknown: 0,
       total: 0,
+      status: 'unknown',
       lineStart: suite.lineStart,
       lineEnd: suite.lineEnd,
       import: suite.import,
       classId: suite.classId,
+      sourceHash: suite.sourceHash,
       duration: 0,
-      tests: []
+      tests: {}
     };
   }
 
@@ -118,12 +122,14 @@ export class TestExecutor {
       methodName: test.methodName,
       description: test.description,
       classId: test.classId,
+      tags: test.tags,
       lineStart: test.lineStart,
       lineEnd: test.lineEnd,
       lineBodyStart: test.lineBodyStart,
       import: test.import,
       sourceImport: test.sourceImport,
-      status: 'skipped',
+      sourceHash: test.sourceHash,
+      status: 'unknown',
       assertions: [],
       duration: 0,
       durationTotal: 0,
@@ -184,19 +190,23 @@ export class TestExecutor {
     }
 
     const result: SuiteResult = this.createSuiteResult(suite);
+    const validTestMethodNames = new Set(tests.map(t => t.methodName));
+    const testConfigs = Object.fromEntries(
+      Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key))
+    );
 
     const startTime = Date.now();
 
     // Mark suite start
-    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite });
+    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite: { ...suite, tests: testConfigs } });
 
-    const mgr = new TestPhaseManager(suite, result, e => this.#onSuiteFailure(e));
+    const manager = new TestPhaseManager(suite, result, event => this.#onSuiteFailure(event));
 
     const originalEnv = { ...process.env };
 
     try {
       // Handle the BeforeAll calls
-      await mgr.startPhase('all');
+      await manager.startPhase('all');
 
       const suiteEnv = { ...process.env };
 
@@ -212,29 +222,30 @@ export class TestExecutor {
         const testStart = Date.now();
 
         // Handle BeforeEach
-        await mgr.startPhase('each');
+        await manager.startPhase('each');
 
         // Run test
         const testResult = await this.executeTest(test);
         result[testResult.status]++;
-        result.tests.push(testResult);
+        result.tests[testResult.methodName] = testResult;
 
         // Handle after each
-        await mgr.endPhase('each');
+        await manager.endPhase('each');
         testResult.durationTotal = Date.now() - testStart;
       }
 
       // Handle after all
-      await mgr.endPhase('all');
-    } catch (err) {
-      await mgr.onError(err);
+      await manager.endPhase('all');
+    } catch (error) {
+      await manager.onError(error);
     }
 
     // Restore env
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-    result.total = result.passed + result.failed;
+    result.total = result.passed + result.failed + result.skipped;
+    result.status = TestModelUtil.countsToTestStatus(result);
 
     // Mark suite complete
     this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });
@@ -246,12 +257,12 @@ export class TestExecutor {
   async execute(run: TestRun): Promise<void> {
     try {
       await Runtime.importFrom(run.import);
-    } catch (err) {
-      if (!(err instanceof Error)) {
-        throw err;
+    } catch (error) {
+      if (!(error instanceof Error)) {
+        throw error;
       }
-      console.error(err);
-      this.#onSuiteFailure(AssertUtil.gernerateImportFailure(run.import, err));
+      console.error(error);
+      this.#onSuiteFailure(AssertUtil.gernerateImportFailure(run.import, error));
       return;
     }
 

package/src/execute/phase.ts

@@ -8,7 +8,7 @@ class TestBreakout extends Error {
   source?: Error;
 }
 
-const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.val) ?? 15000;
+const TEST_PHASE_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_PHASE_TIMEOUT.value) ?? 15000;
 
 /**
  * Test Phase Execution Manager.
@@ -67,9 +67,9 @@ export class TestPhaseManager {
   /**
    * On error, handle stubbing out error for the phases in progress
    */
-  async onError(err: Error | unknown): Promise<void> {
-    if (!(err instanceof Error)) {
-      throw err;
+  async onError(error: Error | unknown): Promise<void> {
+    if (!(error instanceof Error)) {
+      throw error;
     }
 
     for (const ph of this.#progress) {
@@ -82,12 +82,12 @@ export class TestPhaseManager {
 
     const failure = AssertUtil.generateSuiteFailure(
       this.#suite,
-      err instanceof TestBreakout ? err.message : 'all',
-      err instanceof TestBreakout ? err.source! : err
+      error instanceof TestBreakout ? error.message : 'all',
+      error instanceof TestBreakout ? error.source! : error
     );
 
     this.#onSuiteFailure(failure);
-    this.#result.tests.push(failure.testResult);
+    this.#result.tests[failure.testResult.methodName] = failure.testResult;
     this.#result.failed++;
   }
 }

package/src/execute/run.ts

@@ -0,0 +1,247 @@
+import { createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+import readline from 'node:readline/promises';
+import path from 'node:path';
+
+import { Env, ExecUtil, ShutdownManager, Util, RuntimeIndex, Runtime, TimeUtil, JSONUtil } from '@travetto/runtime';
+import { WorkPool } from '@travetto/worker';
+import { Registry } from '@travetto/registry';
+
+import type { TestConfig, TestRunInput, TestRun, TestGlobInput, TestDiffInput } from '../model/test.ts';
+import type { TestRemoveEvent } from '../model/event.ts';
+import type { TestConsumerShape } from '../consumer/types.ts';
+import { RunnableTestConsumer } from '../consumer/types/runnable.ts';
+import type { TestConsumerConfig } from './types.ts';
+import { TestConsumerRegistryIndex } from '../consumer/registry-index.ts';
+import { TestExecutor } from './executor.ts';
+import { buildStandardTestManager } from '../worker/standard.ts';
+import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+
+type RunState = {
+  runs: TestRun[];
+  removes?: TestRemoveEvent[];
+};
+
+/**
+ * Test Utilities for Running
+ */
+export class RunUtil {
+  /**
+   * Add 50 ms to the shutdown to allow for buffers to output properly
+   */
+  static registerCleanup(scope: string): void {
+    ShutdownManager.onGracefulShutdown(() => Util.blockingTimeout(50), `test.${scope}.bufferOutput`);
+  }
+
+  /**
+   * Determine if a given file path is a valid test file
+   */
+  static async isTestFile(file: string): Promise<boolean> {
+    const reader = readline.createInterface({ input: createReadStream(file) });
+    const state = { imp: false, suite: false };
+    for await (const line of reader) {
+      state.imp ||= line.includes('@travetto/test');
+      state.suite ||= line.includes('Suite'); // Decorator or name
+      if (state.imp && state.suite) {
+        reader.close();
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Find all valid test files given the globs
+   */
+  static async* getTestImports(globs?: string[]): AsyncIterable<string> {
+    const all = RuntimeIndex.find({
+      module: mod => mod.roles.includes('test') || mod.roles.includes('std'),
+      folder: folder => folder === 'test',
+      file: file => file.role === 'test'
+    });
+
+    // Collect globs
+    if (globs?.length) {
+      const allFiles = new Map(all.map(file => [file.sourceFile, file]));
+      for await (const item of fs.glob(globs)) {
+        const source = Runtime.workspaceRelative(path.resolve(item));
+        const match = allFiles.get(source);
+        if (match && await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    } else {
+      for await (const match of all) {
+        if (await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    }
+  }
+
+  /**
+   * Get count of tests for a given set of globs
+   * @param input
+   */
+  static async resolveGlobInput({ globs, tags, metadata }: TestGlobInput): Promise<TestRun[]> {
+    const digestProcess = await ExecUtil.getResult(
+      ExecUtil.spawnTrv('test:digest', ['-o', 'json', ...globs], {
+        env: { ...process.env, ...Env.FORCE_COLOR.export(0), ...Env.NO_COLOR.export(true) },
+      }),
+      { catch: true }
+    );
+
+    if (!digestProcess.valid) {
+      throw new Error(digestProcess.stderr);
+    }
+
+    const testFilter = tags?.length ?
+      Util.allowDeny<string, [TestConfig]>(
+        tags,
+        rule => rule,
+        (rule, core) => core.tags?.includes(rule) ?? false
+      ) :
+      ((): boolean => true);
+
+    const parsed: TestConfig[] = JSONUtil.parseSafe(digestProcess.stdout);
+
+    const events = parsed.filter(testFilter).reduce((runs, test) => {
+      if (!runs.has(test.classId)) {
+        runs.set(test.classId, { import: test.import, classId: test.classId, methodNames: [], runId: Util.uuid(), metadata });
+      }
+      runs.get(test.classId)!.methodNames!.push(test.methodName);
+      return runs;
+    }, new Map<string, TestRun>());
+
+    return [...events.values()].sort((a, b) => a.runId!.localeCompare(b.runId!));
+  }
+
+  /**
+   * Resolve a test diff source to ensure we are only running changed tests
+   */
+  static async resolveDiffInput({ import: importPath, diffSource: diff, metadata }: TestDiffInput): Promise<RunState> {
+    // Runs, defaults to new classes
+    const runs: TestRun[] = [];
+    const addRun = (clsId: string | undefined, methods?: string[]): void => {
+      runs.push({ import: importPath, classId: clsId, methodNames: methods?.length ? methods : undefined, metadata });
+    };
+    const removes: TestRemoveEvent[] = [];
+    const removeTest = (clsId: string, methodName?: string): void => {
+      removes.push({ type: 'removeTest', import: importPath, classId: clsId, methodName });
+    };
+
+    const imported = await Registry.manualInit([importPath]);
+    const classes = Object.fromEntries(
+      imported
+        .filter(cls => SuiteRegistryIndex.hasConfig(cls))
+        .map(cls => [cls.Ⲑid, SuiteRegistryIndex.getConfig(cls)])
+    );
+
+    // New classes
+    for (const clsId of Object.keys(classes)) {
+      if (!diff[clsId]) {
+        addRun(clsId);
+      }
+    }
+
+    // Looking at Diff
+    for (const [clsId, config] of Object.entries(diff)) {
+      const local = classes[clsId];
+      if (!local) { // Removed classes
+        removeTest(clsId);
+      } else if (local.sourceHash !== config.sourceHash) { // Class changed or added
+        // Methods to run, defaults to newly added
+        const methods: string[] = Object.keys(local.tests ?? {}).filter(key => !config.methods[key]);
+        let didRemove = false;
+        for (const key of Object.keys(config.methods)) {
+          const localMethod = local.tests?.[key];
+          if (!localMethod) { // Test is removed
+            removeTest(clsId, key);
+            didRemove = true;
+          } else if (localMethod.sourceHash !== config.methods[key]) { // Method changed or added
+            methods.push(key);
+          }
+        }
+        if (!didRemove || methods.length > 0) {
+          addRun(clsId, methods);
+        }
+      }
+    }
+
+    if (runs.length === 0 && removes.length === 0) { // Re-run entire file, classes unchanged
+      addRun(undefined);
+    }
+
+    return { runs, removes };
+  }
+
+  /**
+   * Reinitialize the manifest if needed, mainly for single test runs
+   */
+  static async reinitManifestIfNeeded(runs: TestRun[]): Promise<void> {
+    if (runs.length === 1) {
+      const [run] = runs;
+      const entry = RuntimeIndex.getFromImport(run.import)!;
+
+      if (entry.module !== Runtime.main.name) {
+        RuntimeIndex.reinitForModule(entry.module);
+      }
+    }
+  }
+
+  /**
+   * Build test consumer that wraps a given targeted consumer, and the tests to be run
+   */
+  static async getRunnableConsumer(target: TestConsumerShape, testRuns: TestRun[]): Promise<RunnableTestConsumer> {
+    const consumer = new RunnableTestConsumer(target);
+    const testCount = testRuns.reduce((acc, cur) => acc + (cur.methodNames ? cur.methodNames.length : 0), 0);
+
+    await consumer.onStart({ testCount });
+    return consumer;
+  }
+
+  /**
+   * Resolve input into run state
+   */
+  static async resolveInput(input: TestRunInput): Promise<RunState> {
+    if ('diffSource' in input) {
+      return await this.resolveDiffInput(input);
+    } else if ('globs' in input) {
+      return { runs: await this.resolveGlobInput(input) };
+    } else {
+      return { runs: [input], removes: [] };
+    }
+  }
+
+  /**
+   * Run tests
+   */
+  static async runTests(consumerConfig: TestConsumerConfig, input: TestRunInput): Promise<boolean | undefined> {
+    const { runs, removes } = await this.resolveInput(input);
+
+    await this.reinitManifestIfNeeded(runs);
+
+    const targetConsumer = await TestConsumerRegistryIndex.getInstance(consumerConfig);
+    const consumer = await this.getRunnableConsumer(targetConsumer, runs);
+
+    for (const item of removes ?? []) {
+      consumer.onRemoveEvent(item);
+    }
+
+    if (runs.length === 1) {
+      await new TestExecutor(consumer).execute(runs[0]);
+    } else {
+      await WorkPool.run(
+        run => buildStandardTestManager(consumer, run),
+        runs,
+        {
+          idleTimeoutMillis: TimeUtil.asMillis(10, 's'),
+          min: 1,
+          max: consumerConfig.concurrency
+        }
+      );
+    }
+
+    return consumer.summarizeAsBoolean();
+  }
+}

package/src/execute/types.ts

@@ -1,9 +1,7 @@
-import { TestRun } from '../model/test.ts';
-
 /**
- * Run state
+ * Test Consumer Configuration
  */
-export interface RunState {
+export interface TestConsumerConfig {
   /**
    * Test result consumer
    */
@@ -16,17 +14,4 @@ export interface RunState {
    * Number of test suites to run concurrently, when mode is not single
    */
   concurrency?: number;
-  /**
-   * The tags to include or exclude from testing
-   */
-  tags?: string[];
-  /**
-   * target
-   */
-  target: TestRun | {
-    /**
-     * Globs to run
-     */
-    globs?: string[];
-  };
 }