@travetto/test 5.0.0-rc.0 → 5.0.0-rc.10

package/src/execute/promise.ts

@@ -1,7 +1,8 @@
 import { createHook, executionAsyncId } from 'node:async_hooks';
+import { isPromise } from 'node:util/types';
 
 import { ExecutionError } from '@travetto/worker';
-import { Util } from '@travetto/base';
+import { Util } from '@travetto/runtime';
 
 /**
  * Promise watcher, to catch any unfinished promises
@@ -11,9 +12,8 @@ export class PromiseCapturer {
   #id: number = 0;
 
   #init(id: number, type: string, triggerId: number, resource: unknown): void {
-    if (this.#id && type === 'PROMISE' && triggerId === this.#id) {
-      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-      this.#pending.set(id, resource as Promise<unknown>);
+    if (this.#id && type === 'PROMISE' && triggerId === this.#id && isPromise(resource)) {
+      this.#pending.set(id, resource);
     }
   }
 

package/src/execute/runner.ts

@@ -1,7 +1,6 @@
 import path from 'node:path';
 
-import { path as mp, RuntimeContext, RuntimeIndex } from '@travetto/manifest';
-import { TimeUtil } from '@travetto/base';
+import { TimeUtil, Runtime, RuntimeIndex } from '@travetto/runtime';
 import { WorkPool } from '@travetto/worker';
 
 import { buildStandardTestManager } from '../worker/standard';
@@ -10,6 +9,7 @@ import { RunnableTestConsumer } from '../consumer/types/runnable';
 import { TestExecutor } from './executor';
 import { RunnerUtil } from './util';
 import { RunState } from './types';
+import { TestConfig, TestRun } from '../model/test';
 
 /**
  * Test Runner
@@ -22,29 +22,26 @@ export class Runner {
     this.#state = state;
   }
 
-  get patterns(): RegExp[] {
-    return this.#state.args.map(x => new RegExp(mp.toPosix(x)));
-  }
-
   /**
    * Run all files
    */
-  async runFiles(): Promise<boolean> {
+  async runFiles(globs?: string[]): Promise<boolean> {
     const consumer = await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format);
 
-    const files = (await RunnerUtil.getTestFiles(this.patterns)).map(f => f.sourceFile);
+    console.debug('Running', { globs });
 
-    console.debug('Running', { files, patterns: this.patterns });
+    const tests = await RunnerUtil.getTestDigest(globs, this.#state.tags);
+    const testRuns = RunnerUtil.getTestRuns(tests)
+      .sort((a, b) => a.runId!.localeCompare(b.runId!));
 
-    const testCount = await RunnerUtil.getTestCount(this.#state.args);
-    await consumer.onStart({ testCount });
+    await consumer.onStart({ testCount: tests.length });
     await WorkPool.run(
-      buildStandardTestManager.bind(null, consumer),
-      files,
+      f => buildStandardTestManager(consumer, f),
+      testRuns,
       {
         idleTimeoutMillis: TimeUtil.asMillis(10, 's'),
         min: 1,
-        max: this.#state.concurrency,
+        max: this.#state.concurrency
       });
 
     return consumer.summarizeAsBoolean();
@@ -53,18 +50,30 @@ export class Runner {
   /**
    * Run a single file
    */
-  async runSingle(): Promise<boolean> {
-    const mod = RuntimeIndex.getEntry(path.resolve(this.#state.args[0]))!;
-    if (mod.module !== RuntimeContext.main.name) {
-      RuntimeIndex.reinitForModule(mod.module);
+  async runSingle(run: TestRun): Promise<boolean> {
+    run.import =
+      RuntimeIndex.getFromImport(run.import)?.import ??
+      RuntimeIndex.getFromSource(path.resolve(run.import))?.import!;
+
+    const entry = RuntimeIndex.getFromImport(run.import)!;
+
+    if (entry.module !== Runtime.main.name) {
+      RuntimeIndex.reinitForModule(entry.module);
     }
 
-    const consumer = await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format);
+    const filter = (run.methodNames?.length) ?
+      (cfg: TestConfig): boolean => run.methodNames!.includes(cfg.methodName) :
+      undefined;
 
-    const [file, ...args] = this.#state.args;
+    const consumer = (await RunnableTestConsumer.get(this.#state.consumer ?? this.#state.format))
+      .withTransformer(e => {
+        // Copy run metadata to event
+        e.metadata = run.metadata;
+        return e;
+      });
 
     await consumer.onStart({});
-    await TestExecutor.execute(consumer, file, ...args);
+    await new TestExecutor(consumer, filter).execute(run);
     return consumer.summarizeAsBoolean();
   }
 
@@ -72,9 +81,10 @@ export class Runner {
    * Run the runner, based on the inputs passed to the constructor
    */
   async run(): Promise<boolean | undefined> {
-    switch (this.#state.mode) {
-      case 'single': return await this.runSingle();
-      case 'standard': return await this.runFiles();
+    if ('import' in this.#state.target) {
+      return await this.runSingle(this.#state.target);
+    } else {
+      return await this.runFiles(this.#state.target.globs);
     }
   }
 }

package/src/execute/types.ts

@@ -1,4 +1,5 @@
 import { TestConsumer } from '../consumer/types';
+import { TestRun } from '../model/test';
 
 /**
  * Run state
@@ -13,19 +14,20 @@ export interface RunState {
    */
   consumer?: TestConsumer;
   /**
-   * Test mode
-   */
-  mode?: 'single' | 'watch' | 'standard';
-  /**
-   * Show progress to stderr
+   * Number of test suites to run concurrently, when mode is not single
    */
-  showProgress?: boolean;
+  concurrency?: number;
   /**
-   * Number of test suites to run concurrently, when mode is not single
+   * The tags to include or exclude from testing
    */
-  concurrency: number;
+  tags?: string[];
   /**
-   * Input arguments
+   * target
    */
-  args: string[];
+  target: TestRun | {
+    /**
+     * Globs to run
+     */
+    globs?: string[];
+  };
 }

package/src/execute/util.ts

@@ -1,9 +1,10 @@
 import { spawn } from 'node:child_process';
 import { createReadStream } from 'node:fs';
-import readline from 'node:readline';
+import fs from 'node:fs/promises';
+import readline from 'node:readline/promises';
 
-import { Env, ExecUtil, ShutdownManager, Util } from '@travetto/base';
-import { IndexedFile, RuntimeIndex } from '@travetto/manifest';
+import { Env, ExecUtil, ShutdownManager, Util, RuntimeIndex, Runtime } from '@travetto/runtime';
+import { TestConfig, TestRun } from '../model/test';
 
 /**
  * Simple Test Utilities
@@ -19,48 +20,57 @@ export class RunnerUtil {
   /**
    * Determine if a given file path is a valid test file
    */
-  static isTestFile(file: string): Promise<boolean> {
-    return new Promise<boolean>((resolve) => {
-      const input = createReadStream(file);
-      const reader = readline.createInterface({ input })
-        .on('line', line => {
-          if (line.includes('@Suite')) {
-            resolve(true);
-            reader.close();
-          }
-        })
-        .on('end', resolve.bind(null, false))
-        .on('close', resolve.bind(null, false));
-    });
+  static async isTestFile(file: string): Promise<boolean> {
+    const reader = readline.createInterface({ input: createReadStream(file) });
+    const state = { imp: false, suite: false };
+    for await (const line of reader) {
+      state.imp ||= line.includes('@travetto/test');
+      state.suite ||= line.includes('Suite'); // Decorator or name
+      if (state.imp && state.suite) {
+        reader.close();
+        return true;
+      }
+    }
+    return false;
   }
 
   /**
    * Find all valid test files given the globs
    */
-  static async getTestFiles(globs?: RegExp[]): Promise<IndexedFile[]> {
-    const files = RuntimeIndex.find({
+  static async* getTestImports(globs?: string[]): AsyncIterable<string> {
+    const all = RuntimeIndex.find({
       module: m => m.roles.includes('test') || m.roles.includes('std'),
       folder: f => f === 'test',
       file: f => f.role === 'test'
-    })
-      .filter(f => globs?.some(g => g.test(f.sourceFile)) ?? true);
-
-    const validFiles = files
-      .map(f => this.isTestFile(f.sourceFile).then(valid => ({ file: f, valid })));
+    });
 
-    return (await Promise.all(validFiles))
-      .filter(x => x.valid)
-      .map(x => x.file);
+    // Collect globs
+    if (globs?.length) {
+      const allFiles = new Map(all.map(x => [x.sourceFile, x]));
+      for await (const item of fs.glob(globs)) {
+        const src = Runtime.workspaceRelative(item);
+        const match = allFiles.get(src);
+        if (match && await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    } else {
+      for await (const match of all) {
+        if (await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    }
   }
 
   /**
-   * Get count of tests for a given set of patterns
-   * @param patterns
+   * Get count of tests for a given set of globs
+   * @param globs
    * @returns
    */
-  static async getTestCount(patterns: string[]): Promise<number> {
+  static async getTestDigest(globs: string[] = ['**/*.ts'], tags?: string[]): Promise<TestConfig[]> {
     const countRes = await ExecUtil.getResult(
-      spawn('npx', ['trv', 'test:count', ...patterns], {
+      spawn('npx', ['trv', 'test:digest', '-o', 'json', ...globs], {
         env: { ...process.env, ...Env.FORCE_COLOR.export(0), ...Env.NO_COLOR.export(true) }
       }),
       { catch: true }
@@ -68,13 +78,30 @@ export class RunnerUtil {
     if (!countRes.valid) {
       throw new Error(countRes.stderr);
     }
-    return countRes.valid ? +countRes.stdout : 0;
+
+    const testFilter = tags?.length ?
+      Util.allowDeny<string, [TestConfig]>(
+        tags,
+        rule => rule,
+        (rule, core) => core.tags?.includes(rule) ?? false
+      ) :
+      ((): boolean => true);
+
+    const res: TestConfig[] = countRes.valid ? JSON.parse(countRes.stdout) : [];
+    return res.filter(testFilter);
   }
 
   /**
-   * Determine if we should invoke the debugger
+   * Get run events
    */
-  static get tryDebugger(): boolean {
-    return Env.TRV_TEST_BREAK_ENTRY.isTrue;
+  static getTestRuns(tests: TestConfig[]): TestRun[] {
+    const events = tests.reduce((acc, test) => {
+      if (!acc.has(test.classId)) {
+        acc.set(test.classId, { import: test.import, classId: test.classId, methodNames: [], runId: Util.uuid() });
+      }
+      acc.get(test.classId)!.methodNames!.push(test.methodName);
+      return acc;
+    }, new Map<string, TestRun>());
+    return [...events.values()];
   }
 }

package/src/execute/watcher.ts

@@ -1,25 +1,14 @@
 import { RootRegistry, MethodSource } from '@travetto/registry';
-import { WorkPool, WorkQueue } from '@travetto/worker';
-import { RuntimeIndex } from '@travetto/manifest';
+import { WorkPool } from '@travetto/worker';
+import { AsyncQueue, Runtime, RuntimeIndex, castTo, describeFunction } from '@travetto/runtime';
 
 import { SuiteRegistry } from '../registry/suite';
 import { buildStandardTestManager } from '../worker/standard';
 import { TestConsumerRegistry } from '../consumer/registry';
 import { CumulativeSummaryConsumer } from '../consumer/types/cumulative';
-import { RunEvent } from '../worker/types';
+import { TestRun } from '../model/test';
 import { RunnerUtil } from './util';
-import { TestEvent } from '../model/event';
-
-function isRunEvent(ev: unknown): ev is RunEvent {
-  return typeof ev === 'object' && !!ev && 'type' in ev && typeof ev.type === 'string' && ev.type === 'run-test';
-}
-
-export type TestWatchEvent =
-  TestEvent |
-  { type: 'removeTest', method: string, file: string, classId: string } |
-  { type: 'ready' } |
-  { type: 'log', message: string };
-
+import { TestReadyEvent, TestRemovedEvent } from '../worker/types';
 /**
  * Test Watcher.
  *
@@ -33,16 +22,24 @@ export class TestWatcher {
   static async watch(format: string, runAllOnStart = true): Promise<void> {
     console.debug('Listening for changes');
 
-    const itr = new WorkQueue<string>();
-
     await SuiteRegistry.init();
     SuiteRegistry.listen(RootRegistry);
+    await RootRegistry.init();
+
+    const events: TestRun[] = [];
+
+    if (runAllOnStart) {
+      const tests = await RunnerUtil.getTestDigest();
+      events.push(...RunnerUtil.getTestRuns(tests));
+    }
 
-    const consumer = new CumulativeSummaryConsumer(await TestConsumerRegistry.getInstance(format));
+    const itr = new AsyncQueue(events);
+    const consumer = new CumulativeSummaryConsumer(await TestConsumerRegistry.getInstance(format))
+      .withFilter(x => x.metadata?.partial !== true || x.type !== 'suite');
 
     new MethodSource(RootRegistry).on(e => {
       const [cls, method] = (e.prev ?? e.curr ?? []);
-      if (!cls || RuntimeIndex.getFunctionMetadata(cls)?.abstract) {
+      if (!cls || describeFunction(cls).abstract) {
         return;
       }
       if (!method) {
@@ -52,39 +49,40 @@ export class TestWatcher {
       const conf = SuiteRegistry.getByClassAndMethod(cls, method)!;
       if (e.type !== 'removing') {
         if (conf) {
-          const key = `${conf.file}#${conf.class.name}#${conf.methodName}`;
-          itr.add(key, true); // Shift to front
+          const run: TestRun = {
+            import: conf.import, classId: conf.classId, methodNames: [conf.methodName], metadata: { partial: true }
+          };
+          console.log('Triggering', run);
+          itr.add(run, true); // Shift to front
         }
       } else {
         process.send?.({
           type: 'removeTest',
+          methodNames: method?.name ? [method.name!] : undefined!,
           method: method?.name,
           classId: cls?.Ⲑid,
-          file: RuntimeIndex.getFunctionMetadata(cls)?.source
-        });
+          import: Runtime.getImport(cls)
+        } satisfies TestRemovedEvent);
       }
     });
 
-    // If a file is changed, but doesn't emit classes, re-run whole file
-    RootRegistry.onNonClassChanges(file => itr.add(file));
 
-    await RootRegistry.init();
+    // If a file is changed, but doesn't emit classes, re-run whole file
+    RootRegistry.onNonClassChanges(imp => itr.add({ import: imp }));
 
     process.on('message', ev => {
-      if (isRunEvent(ev)) {
+      if (typeof ev === 'object' && ev && 'type' in ev && ev.type === 'run-test') {
+        console.log('Received message', ev);
+        // Legacy
+        if ('file' in ev && typeof ev.file === 'string') {
+          ev = { import: RuntimeIndex.getFromSource(ev.file)?.import! };
+        }
         console.debug('Manually triggered', ev);
-        itr.add([ev.file, ev.class, ev.method].filter(x => !!x).join('#'), true);
+        itr.add(castTo(ev), true);
       }
     });
 
-    process.send?.({ type: 'ready' });
-
-    if (runAllOnStart) {
-      for (const test of await RunnerUtil.getTestFiles()) {
-        await import(test.import);
-        itr.add(test.sourceFile);
-      }
-    }
+    process.send?.({ type: 'ready' } satisfies TestReadyEvent);
 
     await WorkPool.run(
       buildStandardTestManager.bind(null, consumer),

package/src/fixture.ts

@@ -1,7 +1,12 @@
-import { FileLoader } from '@travetto/base';
+import { FileLoader, Runtime } from '@travetto/runtime';
 
 export class TestFixtures extends FileLoader {
   constructor(modules: string[] = []) {
-    super(['@#test/fixtures', ...['@', ...modules.flat(), '@@'].map(x => `${x}#support/fixtures`)]);
+    super([
+      '@#test/fixtures',
+      '@#support/fixtures',
+      ...modules.flat().map(x => `${x}#support/fixtures`),
+      '@@#support/fixtures'
+    ].map(v => Runtime.modulePath(v)));
   }
 }

package/src/model/common.ts

@@ -1,14 +1,10 @@
 /** Configuration of a skip */
-export type Skip = boolean | ((instance: unknown) => boolean | Promise<boolean>) | (() => boolean | Promise<boolean>);
+export type Skip = boolean | ((instance: unknown) => boolean | Promise<boolean>);
 
 /**
  * Core Suite definition
  */
 export interface SuiteCore {
-  /**
-   * The module the test is declared in
-   */
-  module: string;
   /**
    * The class id
    */
@@ -18,9 +14,9 @@ export interface SuiteCore {
    */
   description: string;
   /**
-   * It's file
+   * The import location for the suite
    */
-  file: string;
+  import: string;
   /**
    * The first line of the unit
    */
@@ -29,6 +25,10 @@ export interface SuiteCore {
    * The last line of the unit
    */
   lineEnd: number;
+  /**
+   * Tags for a suite or a test
+   */
+  tags?: string[];
 }
 
 /**
@@ -39,4 +39,8 @@ export interface TestCore extends SuiteCore {
    * The first line of the unit body
    */
   lineBodyStart: number;
+  /**
+   * For extended suites, this is location of the actual file where the test exists
+   */
+  sourceImport?: string;
 }

package/src/model/event.ts

@@ -5,17 +5,21 @@ import { SuiteConfig, SuiteResult } from './suite';
  * Targets
  */
 export type EventEntity = 'test' | 'suite' | 'assertion';
+
 /**
  * Phases
  */
 export type EventPhase = 'before' | 'after';
 
+type EventTpl<T extends EventEntity, P extends EventPhase, V extends {}> =
+  { type: T, phase: P, metadata?: Record<string, unknown> } & V;
+
 /**
  * Different test event shapes
  */
 export type TestEvent =
-  { type: 'assertion', phase: 'after', assertion: Assertion } |
-  { type: 'test', phase: 'before', test: TestConfig } |
-  { type: 'test', phase: 'after', test: TestResult } |
-  { type: 'suite', phase: 'before', suite: SuiteConfig } |
-  { type: 'suite', phase: 'after', suite: SuiteResult };
+  EventTpl<'assertion', 'after', { assertion: Assertion }> |
+  EventTpl<'test', 'before', { test: TestConfig }> |
+  EventTpl<'test', 'after', { test: TestResult }> |
+  EventTpl<'suite', 'before', { suite: SuiteConfig }> |
+  EventTpl<'suite', 'after', { suite: SuiteResult }>;

package/src/model/suite.ts

@@ -1,6 +1,6 @@
-import type { Class } from '@travetto/base';
+import type { Class } from '@travetto/runtime';
 
-import { TestConfig, TestResult } from './test';
+import { Assertion, TestConfig, TestResult } from './test';
 import { Skip, SuiteCore } from './common';
 
 /**
@@ -60,9 +60,9 @@ export interface SuiteResult extends Counts {
    */
   classId: string;
   /**
-   * File suite is in
+   * Import for the suite
    */
-  file: string;
+  import: string;
   /**
    * Start of the suite
    */
@@ -80,3 +80,13 @@ export interface SuiteResult extends Counts {
    */
   duration: number;
 }
+
+/**
+ * A total suite failure
+ */
+export interface SuiteFailure {
+  assert: Assertion;
+  testResult: TestResult;
+  test: TestConfig;
+  suite: SuiteConfig;
+}

package/src/model/test.ts

@@ -1,4 +1,4 @@
-import type { Class, TimeSpan } from '@travetto/base';
+import type { Class, TimeSpan } from '@travetto/runtime';
 import { Skip, TestCore } from './common';
 
 export type ThrowableError = string | RegExp | Function;
@@ -62,9 +62,9 @@ export interface Assertion {
    */
   error?: Error;
   /**
-   * File of assertion
+   * Import of assertion
    */
-  file: string;
+  import: string;
   /**
    * Line number
    */
@@ -107,4 +107,30 @@ export interface TestResult extends TestCore {
    * Logging output
    */
   output: Record<string, string>;
-}
+}
+
+/**
+ * Test Run
+ */
+export type TestRun = {
+  /**
+   * Import for run
+   */
+  import: string;
+  /**
+   * Suite class id
+   */
+  classId?: string;
+  /**
+   * Methods names we want to target
+   */
+  methodNames?: string[];
+  /**
+   * Test run metadata
+   */
+  metadata?: Record<string, unknown>;
+  /**
+   * unique id for the run
+   */
+  runId?: string;
+};