@travetto/test 5.0.0-rc.8 → 5.0.0

package/src/execute/watcher.ts

@@ -1,27 +1,14 @@
 import { RootRegistry, MethodSource } from '@travetto/registry';
-import { WorkPool, WorkQueue } from '@travetto/worker';
-import { Runtime, describeFunction } from '@travetto/runtime';
+import { WorkPool } from '@travetto/worker';
+import { AsyncQueue, Runtime, RuntimeIndex, castTo, describeFunction } from '@travetto/runtime';
 
 import { SuiteRegistry } from '../registry/suite';
 import { buildStandardTestManager } from '../worker/standard';
 import { TestConsumerRegistry } from '../consumer/registry';
 import { CumulativeSummaryConsumer } from '../consumer/types/cumulative';
-import { RunRequest } from '../worker/types';
+import { TestRun } from '../model/test';
 import { RunnerUtil } from './util';
-import { TestEvent } from '../model/event';
-
-function isRunRequest(ev: unknown): ev is RunRequest {
-  return typeof ev === 'object' && !!ev && 'type' in ev && typeof ev.type === 'string' && ev.type === 'run-test';
-}
-
-type RemoveTestEvent = { type: 'removeTest', method: string, import: string, classId: string };
-
-export type TestWatchEvent =
-  TestEvent |
-  RemoveTestEvent |
-  { type: 'ready' } |
-  { type: 'log', message: string };
-
+import { TestReadyEvent, TestRemovedEvent } from '../worker/types';
 /**
  * Test Watcher.
  *
@@ -35,12 +22,20 @@ export class TestWatcher {
   static async watch(format: string, runAllOnStart = true): Promise<void> {
     console.debug('Listening for changes');
 
-    const itr = new WorkQueue<string | RunRequest>();
-
     await SuiteRegistry.init();
     SuiteRegistry.listen(RootRegistry);
+    await RootRegistry.init();
+
+    const events: TestRun[] = [];
+
+    if (runAllOnStart) {
+      const tests = await RunnerUtil.getTestDigest();
+      events.push(...RunnerUtil.getTestRuns(tests));
+    }
 
-    const consumer = new CumulativeSummaryConsumer(await TestConsumerRegistry.getInstance(format));
+    const itr = new AsyncQueue(events);
+    const consumer = new CumulativeSummaryConsumer(await TestConsumerRegistry.getInstance(format))
+      .withFilter(x => x.metadata?.partial !== true || x.type !== 'suite');
 
     new MethodSource(RootRegistry).on(e => {
       const [cls, method] = (e.prev ?? e.curr ?? []);
@@ -54,39 +49,40 @@ export class TestWatcher {
       const conf = SuiteRegistry.getByClassAndMethod(cls, method)!;
       if (e.type !== 'removing') {
         if (conf) {
-          const key = { import: conf.import, class: conf.class.name, method: conf.methodName };
-          itr.add(key, true); // Shift to front
+          const run: TestRun = {
+            import: conf.import, classId: conf.classId, methodNames: [conf.methodName], metadata: { partial: true }
+          };
+          console.log('Triggering', run);
+          itr.add(run, true); // Shift to front
         }
       } else {
         process.send?.({
           type: 'removeTest',
+          methodNames: method?.name ? [method.name!] : undefined!,
           method: method?.name,
           classId: cls?.Ⲑid,
           import: Runtime.getImport(cls)
-        } satisfies RemoveTestEvent);
+        } satisfies TestRemovedEvent);
       }
     });
 
-    // If a file is changed, but doesn't emit classes, re-run whole file
-    RootRegistry.onNonClassChanges(imp => itr.add(imp));
 
-    await RootRegistry.init();
+    // If a file is changed, but doesn't emit classes, re-run whole file
+    RootRegistry.onNonClassChanges(imp => itr.add({ import: imp }));
 
     process.on('message', ev => {
-      if (isRunRequest(ev)) {
+      if (typeof ev === 'object' && ev && 'type' in ev && ev.type === 'run-test') {
+        console.log('Received message', ev);
+        // Legacy
+        if ('file' in ev && typeof ev.file === 'string') {
+          ev = { import: RuntimeIndex.getFromSource(ev.file)?.import! };
+        }
         console.debug('Manually triggered', ev);
-        itr.add(ev, true);
+        itr.add(castTo(ev), true);
       }
     });
 
-    process.send?.({ type: 'ready' });
-
-    if (runAllOnStart) {
-      for await (const imp of await RunnerUtil.getTestImports()) {
-        await Runtime.importFrom(imp);
-        itr.add(imp);
-      }
-    }
+    process.send?.({ type: 'ready' } satisfies TestReadyEvent);
 
     await WorkPool.run(
       buildStandardTestManager.bind(null, consumer),

package/src/model/common.ts

@@ -1,5 +1,5 @@
 /** Configuration of a skip */
-export type Skip = boolean | ((instance: unknown) => boolean | Promise<boolean>) | (() => boolean | Promise<boolean>);
+export type Skip = boolean | ((instance: unknown) => boolean | Promise<boolean>);
 
 /**
  * Core Suite definition
@@ -25,6 +25,10 @@ export interface SuiteCore {
    * The last line of the unit
    */
   lineEnd: number;
+  /**
+   * Tags for a suite or a test
+   */
+  tags?: string[];
 }
 
 /**
@@ -35,4 +39,8 @@ export interface TestCore extends SuiteCore {
    * The first line of the unit body
    */
   lineBodyStart: number;
+  /**
+   * For extended suites, this is location of the actual file where the test exists
+   */
+  sourceImport?: string;
 }

package/src/model/event.ts

@@ -5,17 +5,21 @@ import { SuiteConfig, SuiteResult } from './suite';
  * Targets
  */
 export type EventEntity = 'test' | 'suite' | 'assertion';
+
 /**
  * Phases
  */
 export type EventPhase = 'before' | 'after';
 
+type EventTpl<T extends EventEntity, P extends EventPhase, V extends {}> =
+  { type: T, phase: P, metadata?: Record<string, unknown> } & V;
+
 /**
  * Different test event shapes
  */
 export type TestEvent =
-  { type: 'assertion', phase: 'after', assertion: Assertion } |
-  { type: 'test', phase: 'before', test: TestConfig } |
-  { type: 'test', phase: 'after', test: TestResult } |
-  { type: 'suite', phase: 'before', suite: SuiteConfig } |
-  { type: 'suite', phase: 'after', suite: SuiteResult };
+  EventTpl<'assertion', 'after', { assertion: Assertion }> |
+  EventTpl<'test', 'before', { test: TestConfig }> |
+  EventTpl<'test', 'after', { test: TestResult }> |
+  EventTpl<'suite', 'before', { suite: SuiteConfig }> |
+  EventTpl<'suite', 'after', { suite: SuiteResult }>;

package/src/model/suite.ts

@@ -1,6 +1,6 @@
 import type { Class } from '@travetto/runtime';
 
-import { TestConfig, TestResult } from './test';
+import { Assertion, TestConfig, TestResult } from './test';
 import { Skip, SuiteCore } from './common';
 
 /**
@@ -80,3 +80,13 @@ export interface SuiteResult extends Counts {
    */
   duration: number;
 }
+
+/**
+ * A total suite failure
+ */
+export interface SuiteFailure {
+  assert: Assertion;
+  testResult: TestResult;
+  test: TestConfig;
+  suite: SuiteConfig;
+}

package/src/model/test.ts

@@ -107,4 +107,30 @@ export interface TestResult extends TestCore {
    * Logging output
    */
   output: Record<string, string>;
-}
+}
+
+/**
+ * Test Run
+ */
+export type TestRun = {
+  /**
+   * Import for run
+   */
+  import: string;
+  /**
+   * Suite class id
+   */
+  classId?: string;
+  /**
+   * Methods names we want to target
+   */
+  methodNames?: string[];
+  /**
+   * Test run metadata
+   */
+  metadata?: Record<string, unknown>;
+  /**
+   * unique id for the run
+   */
+  runId?: string;
+};

package/src/registry/suite.ts

@@ -1,8 +1,8 @@
-import { Class, ConcreteClass, Runtime, describeFunction } from '@travetto/runtime';
+import { Class, Runtime, classConstruct, describeFunction, asFull } from '@travetto/runtime';
 import { MetadataRegistry } from '@travetto/registry';
 
 import { SuiteConfig } from '../model/suite';
-import { TestConfig } from '../model/test';
+import { TestConfig, TestRun } from '../model/test';
 
 /**
  * Test Suite registry
@@ -21,6 +21,7 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
     return {
       class: cls,
       classId: cls.Ⲑid,
+      tags: [],
       import: Runtime.getImport(cls),
       lineStart: lines?.[0],
       lineEnd: lines?.[1],
@@ -36,6 +37,7 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
     const lines = describeFunction(cls)?.methods?.[fn.name].lines;
     return {
       class: cls,
+      tags: [],
       import: Runtime.getImport(cls),
       lineStart: lines?.[0],
       lineEnd: lines?.[1],
@@ -57,8 +59,7 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
    * a full projection of all listeners and tests.
    */
   onInstallFinalize<T>(cls: Class<T>): SuiteConfig {
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    const config = this.getOrCreatePending(cls) as SuiteConfig;
+    const config = asFull(this.getOrCreatePending(cls));
     const tests = [...this.pendingFields.get(cls.Ⲑid)!.values()];
 
     const parent = this.getParentClass(cls);
@@ -71,20 +72,19 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
       config.beforeEach.push(...pConf.beforeEach);
       tests.push(...[...pConf.tests.values()].map(t => ({
         ...t,
+        sourceImport: pConf.import,
         class: cls
       })));
     }
 
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    config.instance = new (config.class as ConcreteClass)();
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-    config.tests = tests as TestConfig[];
+    config.instance = classConstruct(config.class);
+    config.tests = tests!.map(x => asFull(x));
+    config.description ||= config.classId;
 
-    if (!config.description) {
-      config.description = config.classId;
-    }
     for (const t of config.tests) {
       t.classId = config.classId;
+      t.import = config.import;
+      t.tags = [...t.tags!, ...config.tags!];
     }
     return config;
   }
@@ -92,9 +92,13 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
   /**
    * Get run parameters from provided input
    */
-  getRunParams(imp: string, clsName?: string, method?: string): { suites: SuiteConfig[] } | { suite: SuiteConfig, test?: TestConfig } {
-    if (clsName && /^\d+$/.test(clsName)) { // If we only have a line number
-      const line = parseInt(clsName, 10);
+  getSuiteTests(run: TestRun): { suite: SuiteConfig, tests: TestConfig[] }[] {
+    const clsId = run.classId;
+    const imp = run.import;
+    const methodNames = run.methodNames ?? [];
+
+    if (clsId && /^\d+$/.test(clsId)) { // If we only have a line number
+      const line = parseInt(clsId, 10);
       const suites = this.getValidClasses()
         .filter(cls => Runtime.getImport(cls) === imp)
         .map(x => this.get(x)).filter(x => !x.skip);
@@ -102,25 +106,25 @@ class $SuiteRegistry extends MetadataRegistry<SuiteConfig, TestConfig> {
 
       if (suite) {
         const test = suite.tests.find(x => line >= x.lineStart && line <= x.lineEnd);
-        return test ? { suite, test } : { suite };
+        return test ? [{ suite, tests: [test] }] : [{ suite, tests: suite.tests }];
       } else {
-        return { suites };
+        return suites.map(x => ({ suite: x, tests: x.tests }));
       }
     } else { // Else lookup directly
-      if (method) {
-        const cls = this.getValidClasses().find(x => x.name === clsName)!;
+      if (methodNames.length) {
+        const cls = this.getValidClasses().find(x => x.Ⲑid === clsId)!;
         const suite = this.get(cls);
-        const test = suite.tests.find(x => x.methodName === method)!;
-        return { suite, test };
-      } else if (clsName) {
-        const cls = this.getValidClasses().find(x => x.name === clsName)!;
+        const tests = suite.tests.filter(x => methodNames.includes(x.methodName))!;
+        return [{ suite, tests }];
+      } else if (clsId) {
+        const cls = this.getValidClasses().find(x => x.Ⲑid === clsId)!;
         const suite = this.get(cls);
-        return { suite };
+        return [{ suite, tests: suite.tests }];
       } else {
         const suites = this.getValidClasses()
           .map(x => this.get(x))
           .filter(x => !describeFunction(x.class).abstract);  // Do not run abstract suites
-        return { suites };
+        return suites.map(x => ({ suite: x, tests: x.tests }));
       }
     }
   }

package/src/trv.d.ts

@@ -12,5 +12,9 @@ declare global {
      * @default 5s
      */
     TRV_TEST_TIMEOUT: TimeSpan | number;
+    /**
+     * The tags to include or exclude during testing
+     */
+    TRV_TEST_TAGS: string[];
   }
 }

package/src/worker/child.ts

@@ -3,16 +3,17 @@ import { createWriteStream } from 'node:fs';
 import { ConsoleManager, Env, Util, Runtime } from '@travetto/runtime';
 import { ChildCommChannel } from '@travetto/worker';
 
-import { ErrorUtil } from '../consumer/error';
+import { SerializeUtil } from '../consumer/serialize';
 import { RunnerUtil } from '../execute/util';
 import { Runner } from '../execute/runner';
-import { Events, RunEvent } from './types';
+import { Events } from './types';
+import { TestRun } from '../model/test';
 
 /**
  * Child Worker for the Test Runner.  Receives events as commands
  * to run specific tests
  */
-export class TestChildWorker extends ChildCommChannel<RunEvent> {
+export class TestChildWorker extends ChildCommChannel<TestRun> {
 
   #done = Util.resolvablePromise();
 
@@ -25,7 +26,7 @@ export class TestChildWorker extends ChildCommChannel<RunEvent> {
         throw err;
       }
       // Mark as errored out
-      this.send(type, { error: ErrorUtil.serializeError(err) });
+      this.send(type, { error: SerializeUtil.serializeError(err) });
     }
   }
 
@@ -56,7 +57,7 @@ export class TestChildWorker extends ChildCommChannel<RunEvent> {
   /**
    * When we receive a command from the parent
    */
-  async onCommand(event: RunEvent & { type: string }): Promise<boolean> {
+  async onCommand(event: TestRun & { type: string }): Promise<boolean> {
     console.debug('on message', { ...event });
 
     if (event.type === Events.INIT) { // On request to init, start initialization
@@ -76,18 +77,11 @@ export class TestChildWorker extends ChildCommChannel<RunEvent> {
   /**
    * Run a specific test/suite
    */
-  async onRunCommand(event: RunEvent): Promise<void> {
-    console.debug('Run');
-
-    console.debug('Running', { import: event.import });
+  async onRunCommand(run: TestRun): Promise<void> {
+    console.debug('Running', { import: run.import });
 
     try {
-      await new Runner({
-        format: 'exec',
-        mode: 'single',
-        args: [event.import, event.class!, event.method!],
-        concurrency: 1
-      }).run();
+      await new Runner({ format: 'exec', target: run }).run();
     } finally {
       this.#done.resolve();
     }

package/src/worker/standard.ts

@@ -3,39 +3,34 @@ import { fork } from 'node:child_process';
 import { Env, RuntimeIndex } from '@travetto/runtime';
 import { ParentCommChannel } from '@travetto/worker';
 
-import { Events, RunEvent, RunRequest } from './types';
+import { Events, TestLogEvent } from './types';
 import { TestConsumer } from '../consumer/types';
-import { ErrorUtil } from '../consumer/error';
+import { SerializeUtil } from '../consumer/serialize';
 import { TestEvent } from '../model/event';
+import { TestRun } from '../model/test';
+
+const log = (message: string): void => {
+  process.send?.({ type: 'log', message } satisfies TestLogEvent);
+};
 
 /**
  *  Produce a handler for the child worker
  */
-export async function buildStandardTestManager(consumer: TestConsumer, imp: string | RunRequest): Promise<void> {
-  let event: RunEvent;
-  process.send?.({ type: 'log', message: `Worker Input ${JSON.stringify(imp)}` });
-
-  if (typeof imp === 'string') {
-    event = { import: imp };
-  } else if ('file' in imp) {
-    event = { import: RuntimeIndex.getFromSource(imp.file)?.import!, class: imp.class, method: imp.method };
-  } else {
-    event = imp;
-  }
-
-  process.send?.({ type: 'log', message: `Worker Executing ${event.import}` });
+export async function buildStandardTestManager(consumer: TestConsumer, run: TestRun): Promise<void> {
+  log(`Worker Input ${JSON.stringify(run)}`);
+  log(`Worker Executing ${run.import}`);
 
-  const { module } = RuntimeIndex.getFromImport(event.import!)!;
-  const suiteMod = RuntimeIndex.getModule(module);
+  const { module } = RuntimeIndex.getFromImport(run.import)!;
+  const suiteMod = RuntimeIndex.getModule(module)!;
 
   const channel = new ParentCommChannel<TestEvent & { error?: Error }>(
     fork(
       RuntimeIndex.resolveFileImport('@travetto/cli/support/entry.trv'), ['test:child'],
       {
-        cwd: suiteMod!.sourcePath,
+        cwd: suiteMod.sourcePath,
         env: {
           ...process.env,
-          ...Env.TRV_MANIFEST.export(suiteMod!.outputPath),
+          ...Env.TRV_MANIFEST.export(suiteMod.outputPath),
           ...Env.TRV_QUIET.export(true)
         },
         stdio: ['ignore', 'ignore', 2, 'ipc']
@@ -58,7 +53,7 @@ export async function buildStandardTestManager(consumer: TestConsumer, imp: stri
   // Listen for child to complete
   const complete = channel.once(Events.RUN_COMPLETE);
   // Start test
-  channel.send(Events.RUN, event);
+  channel.send(Events.RUN, run);
 
   // Wait for complete
   const { error } = await complete;
@@ -66,10 +61,10 @@ export async function buildStandardTestManager(consumer: TestConsumer, imp: stri
   // Kill on complete
   await channel.destroy();
 
-  process.send?.({ type: 'log', message: `Worker Finished ${event.import}` });
+  log(`Worker Finished ${run.import}`);
 
   // If we received an error, throw it
   if (error) {
-    throw ErrorUtil.deserializeError(error);
+    throw SerializeUtil.deserializeError(error);
   }
 }

package/src/worker/types.ts

@@ -1,25 +1,5 @@
-/**
- * Test Run Request
- */
-export type RunRequest = {
-  file: string;
-  class?: string;
-  method?: string;
-} | {
-  import: string;
-  class?: string;
-  method?: string;
-};
-
-/**
- * Test Run Event
- */
-export type RunEvent = {
-  import: string;
-  error?: unknown;
-  class?: string;
-  method?: string;
-};
+import { TestEvent } from '../model/event';
+import { TestRun } from '../model/test';
 
 /**
  * Test Run Event Keys
@@ -30,4 +10,14 @@ export const Events = {
   INIT: 'init',
   INIT_COMPLETE: 'initComplete',
   READY: 'ready'
-};
+};
+
+export type TestRemovedEvent = { type: 'removeTest', method?: string } & TestRun;
+export type TestReadyEvent = { type: 'ready' };
+export type TestLogEvent = { type: 'log', message: string };
+
+export type TestWatchEvent =
+  TestEvent |
+  TestRemovedEvent |
+  TestReadyEvent |
+  TestLogEvent;

package/support/cli.test.ts

@@ -21,6 +21,11 @@ export class TestCommand implements CliCommandShape {
   concurrency: number = WorkPool.DEFAULT_SIZE;
   /** Test run mode */
   mode: TestMode = 'standard';
+  /**
+   * Tags to target or exclude
+   * @alias env.TRV_TEST_TAGS
+   */
+  tags?: string[];
 
   preMain(): void {
     EventEmitter.defaultMaxListeners = 1000;
@@ -48,14 +53,23 @@ export class TestCommand implements CliCommandShape {
     }
   }
 
-  async main(first: string = '**/*', regexes: string[] = []): Promise<void> {
+  async main(first: string = '**/*', globs: string[] = []): Promise<void> {
     const { runTests } = await import('./bin/run');
 
+    const isFirst = await this.isFirstFile(first);
+    const isSingle = this.mode === 'single' || (isFirst && globs.length === 0);
+
     return runTests({
-      args: [first, ...regexes],
-      mode: await this.resolvedMode(first, regexes),
       concurrency: this.concurrency,
-      format: this.format
+      format: this.format,
+      tags: this.tags,
+      target: isSingle ?
+        {
+          import: first,
+          classId: globs[0],
+          methodNames: globs.slice(1),
+        } :
+        { globs: [first, ...globs], }
     });
   }
 }

package/support/{cli.test_count.ts → cli.test_digest.ts}

@@ -5,16 +5,18 @@ import { SuiteRegistry } from '../src/registry/suite';
 import { RunnerUtil } from '../src/execute/util';
 
 @CliCommand({ hidden: true })
-export class TestCountCommand {
+export class TestDigestCommand {
+
+  output: 'json' | 'text' = 'text';
 
   preMain(): void {
     Env.TRV_ROLE.set('test');
     Env.DEBUG.set(false);
   }
 
-  async main(patterns: string[]) {
+  async main(globs: string[] = ['**/*.ts']) {
     // Load all tests
-    for await (const imp of await RunnerUtil.getTestImports(patterns)) {
+    for await (const imp of await RunnerUtil.getTestImports(globs)) {
       try {
         await Runtime.importFrom(imp);
       } catch (err) {
@@ -25,11 +27,17 @@ export class TestCountCommand {
     await SuiteRegistry.init();
 
     const suites = SuiteRegistry.getClasses();
-    const total = suites
+    const all = suites
       .map(c => SuiteRegistry.get(c))
       .filter(c => !describeFunction(c.class).abstract)
-      .reduce((acc, c) => acc + (c.tests?.length ?? 0), 0);
+      .flatMap(c => c.tests);
 
-    console.log(total);
+    if (this.output === 'json') {
+      console.log(JSON.stringify(all));
+    } else {
+      for (const item of all) {
+        console.log(`${item.classId}#${item.methodName}`, item.tags?.join('|') ?? '');
+      }
+    }
   }
 }

package/support/cli.test_direct.ts

@@ -1,4 +1,4 @@
-import { Env } from '@travetto/runtime';
+import { Env, RuntimeIndex } from '@travetto/runtime';
 import { CliCommand } from '@travetto/cli';
 
 import { runTests } from './bin/run';
@@ -17,7 +17,14 @@ export class TestDirectCommand {
     Env.TRV_LOG_TIME.clear();
   }
 
-  main(file: string, args: string[]): Promise<void> {
-    return runTests({ args: [file, ...args], format: this.format, mode: 'single', concurrency: 1 });
+  main(importOrFile: string, clsId?: string, methodsNames: string[] = []): Promise<void> {
+    return runTests({
+      format: this.format,
+      target: {
+        import: importOrFile,
+        classId: clsId,
+        methodNames: methodsNames,
+      }
+    });
   }
 }