@travetto/test 7.0.0-rc.2 → 7.0.0-rc.3

package/src/execute/executor.ts

@@ -3,7 +3,7 @@ import { AssertionError } from 'node:assert';
 import { Env, TimeUtil, Runtime, castTo, classConstruct } from '@travetto/runtime';
 import { Registry } from '@travetto/registry';
 
-import { TestConfig, TestResult, TestRun } from '../model/test.ts';
+import { TestConfig, TestResult, type TestRun } from '../model/test.ts';
 import { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite.ts';
 import { TestConsumerShape } from '../consumer/types.ts';
 import { AssertCheck } from '../assert/check.ts';
@@ -14,6 +14,7 @@ import { AssertUtil } from '../assert/util.ts';
 import { Barrier } from './barrier.ts';
 import { ExecutionError } from './error.ts';
 import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+import { TestModelUtil } from '../model/util.ts';
 
 const TEST_TIMEOUT = TimeUtil.fromValue(Env.TRV_TEST_TIMEOUT.value) ?? 5000;
 
@@ -94,13 +95,16 @@ export class TestExecutor {
       passed: 0,
       failed: 0,
       skipped: 0,
+      unknown: 0,
       total: 0,
+      status: 'unknown',
       lineStart: suite.lineStart,
       lineEnd: suite.lineEnd,
       import: suite.import,
       classId: suite.classId,
+      sourceHash: suite.sourceHash,
       duration: 0,
-      tests: []
+      tests: {}
     };
   }
 
@@ -118,12 +122,14 @@ export class TestExecutor {
       methodName: test.methodName,
       description: test.description,
       classId: test.classId,
+      tags: test.tags,
       lineStart: test.lineStart,
       lineEnd: test.lineEnd,
       lineBodyStart: test.lineBodyStart,
       import: test.import,
       sourceImport: test.sourceImport,
-      status: 'skipped',
+      sourceHash: test.sourceHash,
+      status: 'unknown',
       assertions: [],
       duration: 0,
       durationTotal: 0,
@@ -184,19 +190,23 @@ export class TestExecutor {
     }
 
     const result: SuiteResult = this.createSuiteResult(suite);
+    const validTestMethodNames = new Set(tests.map(t => t.methodName));
+    const testConfigs = Object.fromEntries(
+      Object.entries(suite.tests).filter(([key]) => validTestMethodNames.has(key))
+    );
 
     const startTime = Date.now();
 
     // Mark suite start
-    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite });
+    this.#consumer.onEvent({ phase: 'before', type: 'suite', suite: { ...suite, tests: testConfigs } });
 
-    const mgr = new TestPhaseManager(suite, result, event => this.#onSuiteFailure(event));
+    const manager = new TestPhaseManager(suite, result, event => this.#onSuiteFailure(event));
 
     const originalEnv = { ...process.env };
 
     try {
       // Handle the BeforeAll calls
-      await mgr.startPhase('all');
+      await manager.startPhase('all');
 
       const suiteEnv = { ...process.env };
 
@@ -212,29 +222,30 @@ export class TestExecutor {
         const testStart = Date.now();
 
         // Handle BeforeEach
-        await mgr.startPhase('each');
+        await manager.startPhase('each');
 
         // Run test
         const testResult = await this.executeTest(test);
         result[testResult.status]++;
-        result.tests.push(testResult);
+        result.tests[testResult.methodName] = testResult;
 
         // Handle after each
-        await mgr.endPhase('each');
+        await manager.endPhase('each');
         testResult.durationTotal = Date.now() - testStart;
       }
 
       // Handle after all
-      await mgr.endPhase('all');
+      await manager.endPhase('all');
     } catch (error) {
-      await mgr.onError(error);
+      await manager.onError(error);
     }
 
     // Restore env
     process.env = { ...originalEnv };
 
     result.duration = Date.now() - startTime;
-    result.total = result.passed + result.failed;
+    result.total = result.passed + result.failed + result.skipped;
+    result.status = TestModelUtil.countsToTestStatus(result);
 
     // Mark suite complete
     this.#consumer.onEvent({ phase: 'after', type: 'suite', suite: result });

package/src/execute/phase.ts

@@ -87,7 +87,7 @@ export class TestPhaseManager {
     );
 
     this.#onSuiteFailure(failure);
-    this.#result.tests.push(failure.testResult);
+    this.#result.tests[failure.testResult.methodName] = failure.testResult;
     this.#result.failed++;
   }
 }

package/src/execute/run.ts

@@ -0,0 +1,247 @@
+import { createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+import readline from 'node:readline/promises';
+import path from 'node:path';
+
+import { Env, ExecUtil, ShutdownManager, Util, RuntimeIndex, Runtime, TimeUtil, JSONUtil } from '@travetto/runtime';
+import { WorkPool } from '@travetto/worker';
+import { Registry } from '@travetto/registry';
+
+import type { TestConfig, TestRunInput, TestRun, TestGlobInput, TestDiffInput } from '../model/test.ts';
+import type { TestRemoveEvent } from '../model/event.ts';
+import type { TestConsumerShape } from '../consumer/types.ts';
+import { RunnableTestConsumer } from '../consumer/types/runnable.ts';
+import type { TestConsumerConfig } from './types.ts';
+import { TestConsumerRegistryIndex } from '../consumer/registry-index.ts';
+import { TestExecutor } from './executor.ts';
+import { buildStandardTestManager } from '../worker/standard.ts';
+import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+
+type RunState = {
+  runs: TestRun[];
+  removes?: TestRemoveEvent[];
+};
+
+/**
+ * Test Utilities for Running
+ */
+export class RunUtil {
+  /**
+   * Add 50 ms to the shutdown to allow for buffers to output properly
+   */
+  static registerCleanup(scope: string): void {
+    ShutdownManager.onGracefulShutdown(() => Util.blockingTimeout(50), `test.${scope}.bufferOutput`);
+  }
+
+  /**
+   * Determine if a given file path is a valid test file
+   */
+  static async isTestFile(file: string): Promise<boolean> {
+    const reader = readline.createInterface({ input: createReadStream(file) });
+    const state = { imp: false, suite: false };
+    for await (const line of reader) {
+      state.imp ||= line.includes('@travetto/test');
+      state.suite ||= line.includes('Suite'); // Decorator or name
+      if (state.imp && state.suite) {
+        reader.close();
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Find all valid test files given the globs
+   */
+  static async* getTestImports(globs?: string[]): AsyncIterable<string> {
+    const all = RuntimeIndex.find({
+      module: mod => mod.roles.includes('test') || mod.roles.includes('std'),
+      folder: folder => folder === 'test',
+      file: file => file.role === 'test'
+    });
+
+    // Collect globs
+    if (globs?.length) {
+      const allFiles = new Map(all.map(file => [file.sourceFile, file]));
+      for await (const item of fs.glob(globs)) {
+        const source = Runtime.workspaceRelative(path.resolve(item));
+        const match = allFiles.get(source);
+        if (match && await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    } else {
+      for await (const match of all) {
+        if (await this.isTestFile(match.sourceFile)) {
+          yield match.import;
+        }
+      }
+    }
+  }
+
+  /**
+   * Get count of tests for a given set of globs
+   * @param input
+   */
+  static async resolveGlobInput({ globs, tags, metadata }: TestGlobInput): Promise<TestRun[]> {
+    const digestProcess = await ExecUtil.getResult(
+      ExecUtil.spawnTrv('test:digest', ['-o', 'json', ...globs], {
+        env: { ...process.env, ...Env.FORCE_COLOR.export(0), ...Env.NO_COLOR.export(true) },
+      }),
+      { catch: true }
+    );
+
+    if (!digestProcess.valid) {
+      throw new Error(digestProcess.stderr);
+    }
+
+    const testFilter = tags?.length ?
+      Util.allowDeny<string, [TestConfig]>(
+        tags,
+        rule => rule,
+        (rule, core) => core.tags?.includes(rule) ?? false
+      ) :
+      ((): boolean => true);
+
+    const parsed: TestConfig[] = JSONUtil.parseSafe(digestProcess.stdout);
+
+    const events = parsed.filter(testFilter).reduce((runs, test) => {
+      if (!runs.has(test.classId)) {
+        runs.set(test.classId, { import: test.import, classId: test.classId, methodNames: [], runId: Util.uuid(), metadata });
+      }
+      runs.get(test.classId)!.methodNames!.push(test.methodName);
+      return runs;
+    }, new Map<string, TestRun>());
+
+    return [...events.values()].sort((a, b) => a.runId!.localeCompare(b.runId!));
+  }
+
+  /**
+   * Resolve a test diff source to ensure we are only running changed tests
+   */
+  static async resolveDiffInput({ import: importPath, diffSource: diff, metadata }: TestDiffInput): Promise<RunState> {
+    // Runs, defaults to new classes
+    const runs: TestRun[] = [];
+    const addRun = (clsId: string | undefined, methods?: string[]): void => {
+      runs.push({ import: importPath, classId: clsId, methodNames: methods?.length ? methods : undefined, metadata });
+    };
+    const removes: TestRemoveEvent[] = [];
+    const removeTest = (clsId: string, methodName?: string): void => {
+      removes.push({ type: 'removeTest', import: importPath, classId: clsId, methodName });
+    };
+
+    const imported = await Registry.manualInit([importPath]);
+    const classes = Object.fromEntries(
+      imported
+        .filter(cls => SuiteRegistryIndex.hasConfig(cls))
+        .map(cls => [cls.Ⲑid, SuiteRegistryIndex.getConfig(cls)])
+    );
+
+    // New classes
+    for (const clsId of Object.keys(classes)) {
+      if (!diff[clsId]) {
+        addRun(clsId);
+      }
+    }
+
+    // Looking at Diff
+    for (const [clsId, config] of Object.entries(diff)) {
+      const local = classes[clsId];
+      if (!local) { // Removed classes
+        removeTest(clsId);
+      } else if (local.sourceHash !== config.sourceHash) { // Class changed or added
+        // Methods to run, defaults to newly added
+        const methods: string[] = Object.keys(local.tests ?? {}).filter(key => !config.methods[key]);
+        let didRemove = false;
+        for (const key of Object.keys(config.methods)) {
+          const localMethod = local.tests?.[key];
+          if (!localMethod) { // Test is removed
+            removeTest(clsId, key);
+            didRemove = true;
+          } else if (localMethod.sourceHash !== config.methods[key]) { // Method changed or added
+            methods.push(key);
+          }
+        }
+        if (!didRemove || methods.length > 0) {
+          addRun(clsId, methods);
+        }
+      }
+    }
+
+    if (runs.length === 0 && removes.length === 0) { // Re-run entire file, classes unchanged
+      addRun(undefined);
+    }
+
+    return { runs, removes };
+  }
+
+  /**
+   * Reinitialize the manifest if needed, mainly for single test runs
+   */
+  static async reinitManifestIfNeeded(runs: TestRun[]): Promise<void> {
+    if (runs.length === 1) {
+      const [run] = runs;
+      const entry = RuntimeIndex.getFromImport(run.import)!;
+
+      if (entry.module !== Runtime.main.name) {
+        RuntimeIndex.reinitForModule(entry.module);
+      }
+    }
+  }
+
+  /**
+   * Build test consumer that wraps a given targeted consumer, and the tests to be run
+   */
+  static async getRunnableConsumer(target: TestConsumerShape, testRuns: TestRun[]): Promise<RunnableTestConsumer> {
+    const consumer = new RunnableTestConsumer(target);
+    const testCount = testRuns.reduce((acc, cur) => acc + (cur.methodNames ? cur.methodNames.length : 0), 0);
+
+    await consumer.onStart({ testCount });
+    return consumer;
+  }
+
+  /**
+   * Resolve input into run state
+   */
+  static async resolveInput(input: TestRunInput): Promise<RunState> {
+    if ('diffSource' in input) {
+      return await this.resolveDiffInput(input);
+    } else if ('globs' in input) {
+      return { runs: await this.resolveGlobInput(input) };
+    } else {
+      return { runs: [input], removes: [] };
+    }
+  }
+
+  /**
+   * Run tests
+   */
+  static async runTests(consumerConfig: TestConsumerConfig, input: TestRunInput): Promise<boolean | undefined> {
+    const { runs, removes } = await this.resolveInput(input);
+
+    await this.reinitManifestIfNeeded(runs);
+
+    const targetConsumer = await TestConsumerRegistryIndex.getInstance(consumerConfig);
+    const consumer = await this.getRunnableConsumer(targetConsumer, runs);
+
+    for (const item of removes ?? []) {
+      consumer.onRemoveEvent(item);
+    }
+
+    if (runs.length === 1) {
+      await new TestExecutor(consumer).execute(runs[0]);
+    } else {
+      await WorkPool.run(
+        run => buildStandardTestManager(consumer, run),
+        runs,
+        {
+          idleTimeoutMillis: TimeUtil.asMillis(10, 's'),
+          min: 1,
+          max: consumerConfig.concurrency
+        }
+      );
+    }
+
+    return consumer.summarizeAsBoolean();
+  }
+}

package/src/execute/types.ts

@@ -1,9 +1,7 @@
-import { TestRun } from '../model/test.ts';
-
 /**
- * Run state
+ * Test Consumer Configuration
  */
-export interface RunState {
+export interface TestConsumerConfig {
   /**
    * Test result consumer
    */
@@ -16,17 +14,4 @@ export interface RunState {
    * Number of test suites to run concurrently, when mode is not single
    */
   concurrency?: number;
-  /**
-   * The tags to include or exclude from testing
-   */
-  tags?: string[];
-  /**
-   * target
-   */
-  target: TestRun | {
-    /**
-     * Globs to run
-     */
-    globs?: string[];
-  };
 }

package/src/execute/watcher.ts

@@ -1,14 +1,14 @@
+import { ManifestModuleUtil } from '@travetto/manifest';
 import { Registry } from '@travetto/registry';
 import { WorkPool } from '@travetto/worker';
-import { AsyncQueue, Runtime, RuntimeIndex, castTo, describeFunction } from '@travetto/runtime';
+import { AsyncQueue, RuntimeIndex, TimeUtil, watchCompiler } from '@travetto/runtime';
 
 import { buildStandardTestManager } from '../worker/standard.ts';
 import { TestConsumerRegistryIndex } from '../consumer/registry-index.ts';
 import { CumulativeSummaryConsumer } from '../consumer/types/cumulative.ts';
-import { TestRun } from '../model/test.ts';
-import { RunnerUtil } from './util.ts';
-import { TestReadyEvent, TestRemovedEvent } from '../worker/types.ts';
-import { SuiteRegistryIndex } from '../registry/registry-index.ts';
+import type { TestDiffInput, TestRun } from '../model/test.ts';
+import { RunUtil } from './run.ts';
+import { isTestRunEvent, type TestReadyEvent } from '../worker/types.ts';
 
 /**
  * Test Watcher.
@@ -25,77 +25,51 @@ export class TestWatcher {
 
     await Registry.init();
 
-    const events: TestRun[] = [];
+    const events: (TestRun | TestDiffInput)[] = [];
 
     if (runAllOnStart) {
-      const tests = await RunnerUtil.getTestDigest();
-      events.push(...RunnerUtil.getTestRuns(tests));
+      events.push(...await RunUtil.resolveGlobInput({ globs: [] }));
     }
 
     const queue = new AsyncQueue(events);
     const consumer = new CumulativeSummaryConsumer(
       await TestConsumerRegistryIndex.getInstance({ consumer: format })
-    )
-      .withFilter(event => event.metadata?.partial !== true || event.type !== 'suite');
-
-    Registry.onMethodChange((event) => {
-      const [cls, method] = 'previous' in event ? event.previous : event.current;
-
-      if (!cls || describeFunction(cls).abstract) {
-        return;
-      }
-
-      const classId = cls.Ⲑid;
-      if (!method) {
-        consumer.removeClass(classId);
-        return;
-      }
-
-      const config = SuiteRegistryIndex.getTestConfig(cls, method)!;
-      if (event.type !== 'removing') {
-        if (config) {
-          const run: TestRun = {
-            import: config.import, classId: config.classId, methodNames: [config.methodName], metadata: { partial: true }
-          };
-          console.log('Triggering', run);
-          queue.add(run, true); // Shift to front
-        }
-      } else {
-        process.send?.({
-          type: 'removeTest',
-          methodNames: method?.name ? [method.name!] : undefined!,
-          method: method?.name,
-          classId,
-          import: Runtime.getImport(cls)
-        } satisfies TestRemovedEvent);
-      }
-    }, SuiteRegistryIndex);
-
-    // If a file is changed, but doesn't emit classes, re-run whole file
-    Registry.onNonClassChanges(imp => queue.add({ import: imp }));
+    );
 
     process.on('message', event => {
-      if (typeof event === 'object' && event && 'type' in event && event.type === 'run-test') {
-        console.log('Received message', event);
-        // Legacy
-        if ('file' in event && typeof event.file === 'string') {
-          event = { import: RuntimeIndex.getFromSource(event.file)?.import! };
-        }
-        console.debug('Manually triggered', event);
-        queue.add(castTo(event), true);
+      if (isTestRunEvent(event)) {
+        queue.add(event, true);
       }
     });
 
     process.send?.({ type: 'ready' } satisfies TestReadyEvent);
 
-    await WorkPool.run(
+    const queueProcessor = WorkPool.run(
       buildStandardTestManager.bind(null, consumer),
       queue,
       {
-        idleTimeoutMillis: 120000,
+        idleTimeoutMillis: TimeUtil.asMillis('2m'),
         min: 2,
         max: WorkPool.DEFAULT_SIZE
       }
     );
+
+    for await (const event of watchCompiler()) {
+      const fileType = ManifestModuleUtil.getFileType(event.file);
+      if (
+        (fileType === 'ts' || fileType === 'js') &&
+        RuntimeIndex.findModuleForArbitraryFile(event.file) !== undefined
+      ) {
+        if (event.action === 'delete') {
+          consumer.removeTest(event.import);
+        } else {
+          const diffSource = consumer.produceDiffSource(event.import);
+          queue.add({ import: event.import, diffSource }, true);
+        }
+      }
+    }
+
+    // Cleanup
+    await queueProcessor;
   }
 }

package/src/model/common.ts

@@ -29,6 +29,10 @@ export interface SuiteCore {
    * Description
    */
   description?: string;
+  /**
+   * Hash of the suite/test source code
+   */
+  sourceHash?: number;
 }
 
 /**

package/src/model/event.ts

@@ -14,6 +14,8 @@ export type EventPhase = 'before' | 'after';
 type EventTpl<T extends EventEntity, P extends EventPhase, V extends {}> =
   { type: T, phase: P, metadata?: Record<string, unknown> } & V;
 
+export type TestRemoveEvent = { type: 'removeTest', import: string, classId?: string, methodName?: string, metadata?: Record<string, unknown> };
+
 /**
  * Different test event shapes
  */
@@ -22,4 +24,4 @@ export type TestEvent =
   EventTpl<'test', 'before', { test: TestConfig }> |
   EventTpl<'test', 'after', { test: TestResult }> |
   EventTpl<'suite', 'before', { suite: SuiteConfig }> |
-  EventTpl<'suite', 'after', { suite: SuiteResult }>;
+  EventTpl<'suite', 'after', { suite: SuiteResult }>;

package/src/model/suite.ts

@@ -1,6 +1,6 @@
 import type { Class } from '@travetto/runtime';
 
-import { Assertion, TestConfig, TestResult } from './test.ts';
+import { Assertion, TestConfig, TestResult, type TestStatus } from './test.ts';
 import { Skip, SuiteCore } from './common.ts';
 
 /**
@@ -48,37 +48,26 @@ export interface Counts {
   passed: number;
   skipped: number;
   failed: number;
+  unknown: number;
   total: number;
 }
 
 /**
  * Results of a suite run
  */
-export interface SuiteResult extends Counts {
+export interface SuiteResult extends Counts, SuiteCore {
   /**
-   * Class identifier
+   * All test results
    */
-  classId: string;
-  /**
-   * Import for the suite
-   */
-  import: string;
-  /**
-   * Start of the suite
-   */
-  lineStart: number;
-  /**
-   * End of the suite
-   */
-  lineEnd: number;
-  /**
-   * ALl test results
-   */
-  tests: TestResult[];
+  tests: Record<string, TestResult>;
   /**
    * Suite duration
    */
   duration: number;
+  /**
+   * Overall status
+   */
+  status: TestStatus;
 }
 
 /**

package/src/model/test.ts

@@ -5,6 +5,13 @@ import { Skip, TestCore } from './common.ts';
 export type ThrowableError = string | RegExp | Class<Error> | ((error: Error | string) => boolean | void | undefined);
 export type TestLog = Omit<ConsoleEvent, 'args' | 'scope'> & { message: string };
 
+export type TestDiffSource = Record<string, {
+  sourceHash: number;
+  methods: Record<string, number>;
+}>;
+
+export type TestStatus = 'passed' | 'skipped' | 'failed' | 'unknown';
+
 /**
  * Specific configuration for a test
  */
@@ -88,7 +95,7 @@ export interface TestResult extends TestCore {
   /**
    * status
    */
-  status: 'passed' | 'skipped' | 'failed';
+  status: TestStatus;
   /**
    * Error if failed
    */
@@ -136,3 +143,42 @@ export type TestRun = {
    */
   runId?: string;
 };
+
+/**
+ * Test Diff Input
+ */
+export type TestDiffInput = {
+  /**
+   * Import for run
+   */
+  import: string;
+  /**
+   * Diff Source
+   */
+  diffSource: TestDiffSource;
+  /**
+   * Test run metadata
+   */
+  metadata?: Record<string, unknown>;
+};
+
+/**
+ * Test Glob Input
+ */
+export type TestGlobInput = {
+  /**
+   * Globs to run
+   */
+  globs: string[];
+  /**
+   * Tags to filter by
+   */
+  tags?: string[];
+  /**
+   * Test run metadata
+   */
+  metadata?: Record<string, unknown>;
+};
+
+
+export type TestRunInput = TestRun | TestDiffInput | TestGlobInput;

package/src/model/util.ts

@@ -0,0 +1,8 @@
+import type { Counts } from './suite';
+import type { TestStatus } from './test';
+
+export class TestModelUtil {
+  static countsToTestStatus(counts: Counts): TestStatus {
+    return counts.failed ? 'failed' : (counts.passed ? 'passed' : counts.skipped ? 'skipped' : 'unknown');
+  }
+}