@travetto/test 7.1.4 → 8.0.0-alpha.1

package/README.md

@@ -21,7 +21,7 @@ This module provides unit testing functionality that integrates with the framewo
 **Note**: All tests should be under the `**/*` folders.  The pattern for tests is defined as as a standard glob using [Node](https://nodejs.org)'s built in globbing support.
 
 ## Definition
-A test suite is a collection of individual tests.  All test suites are classes with the [@Suite](https://github.com/travetto/travetto/tree/main/module/test/src/decorator/suite.ts#L15) decorator. Tests are defined as methods on the suite class, using the [@Test](https://github.com/travetto/travetto/tree/main/module/test/src/decorator/test.ts#L25) decorator.  All tests intrinsically support `async`/`await`. 
+A test suite is a collection of individual tests.  All test suites are classes with the [@Suite](https://github.com/travetto/travetto/tree/main/module/test/src/decorator/suite.ts#L13) decorator. Tests are defined as methods on the suite class, using the [@Test](https://github.com/travetto/travetto/tree/main/module/test/src/decorator/test.ts#L25) decorator.  All tests intrinsically support `async`/`await`. 
 
 A simple example would be:
 

package/__index__.ts

@@ -2,6 +2,7 @@ import type { } from './src/trv.d.ts';
 export * from './src/decorator/suite.ts';
 export * from './src/decorator/test.ts';
 export * from './src/model/suite.ts';
+export * from './src/model/error.ts';
 export * from './src/model/test.ts';
 export * from './src/model/event.ts';
 export * from './src/model/util.ts';
@@ -10,5 +11,5 @@ export * from './src/registry/registry-adapter.ts';
 export * from './src/fixture.ts';
 export * from './src/consumer/types.ts';
 export * from './src/consumer/registry-index.ts';
-export * from './src/execute/error.ts';
+
 export { TestWatchEvent } from './src/worker/types.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/test",
-  "version": "7.1.4",
+  "version": "8.0.0-alpha.1",
   "type": "module",
   "description": "Declarative test framework",
   "keywords": [
@@ -28,15 +28,15 @@
     "directory": "module/test"
   },
   "dependencies": {
-    "@travetto/registry": "^7.1.4",
-    "@travetto/runtime": "^7.1.4",
-    "@travetto/terminal": "^7.1.4",
-    "@travetto/worker": "^7.1.4",
+    "@travetto/registry": "^8.0.0-alpha.1",
+    "@travetto/runtime": "^8.0.0-alpha.1",
+    "@travetto/terminal": "^8.0.0-alpha.1",
+    "@travetto/worker": "^8.0.0-alpha.1",
     "yaml": "^2.8.2"
   },
   "peerDependencies": {
-    "@travetto/cli": "^7.1.4",
-    "@travetto/transformer": "^7.1.3"
+    "@travetto/cli": "^8.0.0-alpha.1",
+    "@travetto/transformer": "^8.0.0-alpha.1"
   },
   "peerDependenciesMeta": {
     "@travetto/transformer": {

package/src/assert/capture.ts

@@ -7,6 +7,7 @@ export interface CapturedAssertion extends Partial<Assertion> {
   line: number;
   text: string;
   operator: string;
+  unexpected?: boolean;
 }
 
 /**

package/src/assert/check.ts

@@ -1,19 +1,20 @@
 import assert from 'node:assert';
 import { isPromise } from 'node:util/types';
 
-import { AppError, type Class, castTo, castKey, asConstructable } from '@travetto/runtime';
+import { RuntimeError, type Class, castTo, castKey, asConstructable } from '@travetto/runtime';
 
-import type { ThrowableError, TestConfig, Assertion } from '../model/test.ts';
+import type { ThrowableError, TestConfig, Assertion, TestStatus } from '../model/test.ts';
 import { AssertCapture, type CapturedAssertion } from './capture.ts';
 import { AssertUtil } from './util.ts';
 import { ASSERT_FN_OPERATOR, OP_MAPPING } from './types.ts';
+import { TestExecutionError } from '../model/error.ts';
 
 type StringFields<T> = {
   [K in Extract<keyof T, string>]:
   (T[K] extends string ? K : never)
 }[Extract<keyof T, string>];
 
-const isClass = (input: unknown): input is Class => input === Error || input === AppError || Object.getPrototypeOf(input) !== Object.getPrototypeOf(Function);
+const isClass = (input: unknown): input is Class => input === Error || input === RuntimeError || Object.getPrototypeOf(input) !== Object.getPrototypeOf(Function);
 
 /**
  * Check assertion
@@ -272,18 +273,34 @@ export class AssertCheck {
    * Look for any unhandled exceptions
    */
   static checkUnhandled(test: TestConfig, error: Error | assert.AssertionError): void {
-    let line = AssertUtil.getPositionOfError(error, test.sourceImport ?? test.import).line;
-    if (line === 1) {
-      line = test.lineStart;
-    }
+    const { line } = AssertUtil.getPositionOfError(error) ?? {};
 
     AssertCapture.add({
-      import: test.import,
-      line,
+      import: test.declarationImport ?? test.import,
+      line: line ?? test.lineStart,
       operator: 'throws',
       error,
+      unexpected: true,
       message: error.message,
       text: ('operator' in error ? error.operator : '') || '(uncaught)'
     });
   }
+
+  /**
+   * Validate the test result based on the error and test configuration
+   */
+  static validateTestResultError(test: TestConfig, error: Error | undefined): [TestStatus, Error | undefined] {
+    if (error instanceof assert.AssertionError) {
+      return ['failed', error];
+    } else if (error instanceof TestExecutionError) {
+      this.checkUnhandled(test, error);
+      return ['errored', error];
+    } else if (error === undefined || test.shouldThrow) {
+      error = this.checkError(test.shouldThrow, error); // Rewrite error
+      return [error ? 'failed' : 'passed', error];
+    } else {
+      this.checkUnhandled(test, error);
+      return ['errored', error];
+    }
+  }
 }

package/src/assert/util.ts

@@ -1,10 +1,10 @@
 import util from 'node:util';
 import path from 'node:path';
 
-import { asFull, type Class, hasFunction, Runtime, RuntimeIndex } from '@travetto/runtime';
+import { asFull, type Class, JSONUtil, hasFunction, Runtime, RuntimeIndex, Util } from '@travetto/runtime';
 
-import type { TestConfig, Assertion, TestResult } from '../model/test.ts';
-import type { SuiteConfig, SuiteFailure, SuiteResult } from '../model/suite.ts';
+import type { TestConfig, TestResult } from '../model/test.ts';
+import type { SuiteConfig, SuiteResult } from '../model/suite.ts';
 
 const isCleanable = hasFunction<{ toClean(): unknown }>('toClean');
 
@@ -22,7 +22,7 @@ export class AssertUtil {
         if (isCleanable(value)) {
           return value.toClean();
         } else if (value === null || value.constructor === Object || Array.isArray(value) || value instanceof Date) {
-          return JSON.stringify(value);
+          return JSONUtil.toUTF8(value);
         }
         break;
       }
@@ -39,92 +39,92 @@ export class AssertUtil {
   /**
    * Determine file location for a given error and the stack trace
    */
-  static getPositionOfError(error: Error, importLocation: string): { import: string, line: number } {
-    const workingDirectory = Runtime.mainSourcePath;
-    const lines = (error.stack ?? new Error().stack!)
-      .replace(/[\\/]/g, '/')
-      .split('\n')
-      // Exclude node_modules, target self
-      .filter(lineText => lineText.includes(workingDirectory) && (!lineText.includes('node_modules') || lineText.includes('/support/')));
-
-    const filename = RuntimeIndex.getFromImport(importLocation)?.sourceFile!;
-
-    let best = lines.filter(lineText => lineText.includes(filename))[0];
-
-    if (!best) {
-      [best] = lines.filter(lineText => lineText.includes(`${workingDirectory}/test`));
-    }
-
-    if (!best) {
-      return { import: importLocation, line: 1 };
-    }
-
-    const pth = best.trim().split(/\s+/g).slice(1).pop()!;
-    if (!pth) {
-      return { import: importLocation, line: 1 };
-    }
-
-    const [file, lineNo] = pth
-      .replace(/[()]/g, '')
-      .replace(/^[A-Za-z]:/, '')
-      .split(':');
-
-    let line = parseInt(lineNo, 10);
-    if (Number.isNaN(line)) {
-      line = -1;
-    }
-
-    const outFileParts = file.split(workingDirectory.replace(/^[A-Za-z]:/, ''));
-
-    const outFile = outFileParts.length > 1 ? outFileParts[1].replace(/^[\/]/, '') : filename;
-
-    const result = { import: RuntimeIndex.getFromSource(outFile)?.import!, line };
-
-    return result;
+  static getPositionOfError(error: Error): { import: string, line: number } | undefined {
+    const frames = Util.stackTraceToParts(error.stack ?? new Error().stack!)
+      .map(frame => {
+        const entry = RuntimeIndex.getEntry(frame.filename);
+        return { ...frame, import: entry?.import!, line: entry?.type === 'ts' ? frame.line : 1 };
+      });
+
+    return frames.find(frame => frame.import);
   }
 
   /**
    * Generate a suite error given a suite config, and an error
    */
-  static generateSuiteFailure(suite: SuiteConfig, methodName: string, error: Error): SuiteFailure {
-    const { import: imp, ...rest } = this.getPositionOfError(error, suite.import);
-    let line = rest.line;
-
-    if (line === 1 && suite.lineStart) {
-      line = suite.lineStart;
-    }
-
-    const msg = error.message.split(/\n/)[0];
-
-    const core = { import: imp, classId: suite.classId, methodName, sourceHash: suite.sourceHash };
-    const coreAll = { ...core, description: msg, lineStart: line, lineEnd: line, lineBodyStart: line };
-
-    const assert: Assertion = {
-      ...core,
-      operator: 'throw', error, line, message: msg, text: methodName
-    };
+  static generateSuiteTestFailure(config: { suite: SuiteConfig, test: TestConfig, error: Error, importLocation?: string }): TestResult {
+    const { suite, test, error, importLocation } = config;
+    const testImport = importLocation ?? test.import;
+    const position = this.getPositionOfError(error);
+    const line = position?.line ?? (testImport === suite.import ? suite.lineStart : 1);
     const testResult: TestResult = {
-      ...coreAll,
-      status: 'failed', error, duration: 0, durationTotal: 0, assertions: [assert], output: []
-    };
-    const test: TestConfig = {
-      ...coreAll,
-      class: suite.class, skip: false
+      ...suite.tests[test.methodName],
+      suiteLineStart: suite.lineStart,
+      status: 'errored',
+      error,
+      duration: 0,
+      durationTotal: 0,
+      output: [],
+      assertions: [{
+        import: position?.import ?? testImport,
+        methodName: test.methodName,
+        classId: suite.classId,
+        operator: 'throw',
+        error,
+        line,
+        message: error.message.split(/\n/)[0],
+        text: test.methodName
+      }],
     };
 
-    return { assert, testResult, test, suite };
+    return testResult;
   }
 
   /**
-   * Define import failure as a SuiteFailure object
+   * Generate suite failure
    */
-  static gernerateImportFailure(importLocation: string, error: Error): SuiteFailure {
+  static generateSuiteTestFailures(suite: SuiteConfig, error: Error): TestResult[] {
+    const finalError = error.cause instanceof Error ? error.cause : error;
+    return Object.values(suite.tests).map(test => this.generateSuiteTestFailure({ suite, test, error: finalError }));
+  }
+
+  /**
+   * Define import failure as a TestResult
+   */
+  static gernerateImportFailure(importLocation: string, error: Error): { result: TestResult, test: TestConfig, suite: SuiteResult & SuiteConfig } {
     const name = path.basename(importLocation);
     const classId = `${RuntimeIndex.getFromImport(importLocation)?.id}#${name}`;
     const suite = asFull<SuiteConfig & SuiteResult>({
       class: asFull<Class>({ name }), classId, duration: 0, lineStart: 1, lineEnd: 1, import: importLocation
     });
     error.message = error.message.replaceAll(Runtime.mainSourcePath, '.');
-    return this.generateSuiteFailure(suite, 'require', error);
+    const result = this.generateSuiteTestFailure({
+      suite,
+      test: {
+        methodName: 'require',
+        classId,
+        import: importLocation,
+        class: suite.class,
+        lineBodyStart: 1,
+        lineStart: 1,
+        lineEnd: 1,
+        skip: false
+      },
+      error
+    });
+    const test: TestConfig = {
+      methodName: 'import',
+      classId,
+      import: importLocation,
+      declarationImport: importLocation,
+      lineStart: 0,
+      lineEnd: 0,
+      lineBodyStart: 0,
+      tags: [],
+      description: 'Import Failure',
+      skip: false,
+      class: undefined!
+    };
+    return { result, test, suite };
   }
 }

package/src/consumer/types/cumulative.ts

@@ -56,7 +56,15 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
   onSuiteAfter(result: SuiteResult): SuiteResult {
     // Reset counts
     const suite = this.getSuite(result);
-    const totals: Counts & { duration: number } = { passed: 0, failed: 0, skipped: 0, unknown: 0, total: 0, duration: 0 };
+    const totals: Counts & { duration: number } = {
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      errored: 0,
+      unknown: 0,
+      total: 0,
+      duration: 0
+    };
     for (const test of Object.values(suite.tests)) {
       totals[test.status] += 1;
       totals.total += 1;

package/src/consumer/types/event.ts

@@ -1,9 +1,10 @@
 import type { Writable } from 'node:stream';
 
+import { JSONUtil } from '@travetto/runtime';
+
 import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
-import { CommunicationUtil } from '../../communication.ts';
 
 /**
  * Streams all test events a JSON payload, in an nd-json format
@@ -17,7 +18,7 @@ export class EventStreamer implements TestConsumerShape {
   }
 
   sendPayload(payload: unknown): void {
-    this.#stream.write(`${CommunicationUtil.serialize(payload)}\n`);
+    this.#stream.write(`${JSONUtil.toUTF8(JSONUtil.cloneForTransmit(payload))}\n`);
   }
 
   onEvent(event: TestEvent): void {

package/src/consumer/types/exec.ts

@@ -1,9 +1,9 @@
 import { IpcChannel } from '@travetto/worker';
+import { JSONUtil } from '@travetto/runtime';
 
 import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
-import { CommunicationUtil } from '../../communication.ts';
 
 /**
  * Triggers each event as an IPC command to a parent process
@@ -12,7 +12,7 @@ import { CommunicationUtil } from '../../communication.ts';
 export class ExecutionEmitter extends IpcChannel<TestEvent> implements TestConsumerShape {
 
   sendPayload(payload: unknown & { type: string }): void {
-    this.send(payload.type, CommunicationUtil.serializeToObject(payload));
+    this.send(payload.type, JSONUtil.cloneForTransmit(payload));
   }
 
   onEvent(event: TestEvent): void {

package/src/consumer/types/json.ts

@@ -1,5 +1,7 @@
 import type { Writable } from 'node:stream';
 
+import { JSONUtil } from '@travetto/runtime';
+
 import type { SuitesSummary } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
 
@@ -18,6 +20,6 @@ export class JSONEmitter {
   onEvent(): void { }
 
   onSummary(summary: SuitesSummary): void {
-    this.#stream.write(JSON.stringify(summary, undefined, 2));
+    this.#stream.write(JSONUtil.toUTF8Pretty(summary));
   }
 }

package/src/consumer/types/summarizer.ts

@@ -10,6 +10,7 @@ export class TestResultsSummarizer implements TestConsumerShape {
   summary: SuitesSummary = {
     passed: 0,
     failed: 0,
+    errored: 0,
     skipped: 0,
     unknown: 0,
     total: 0,
@@ -21,6 +22,7 @@ export class TestResultsSummarizer implements TestConsumerShape {
   #merge(result: SuiteResult): void {
     this.summary.suites.push(result);
     this.summary.failed += result.failed;
+    this.summary.errored += result.errored;
     this.summary.passed += result.passed;
     this.summary.unknown += result.unknown;
     this.summary.skipped += result.skipped;

package/src/consumer/types/tap-summary.ts

@@ -2,13 +2,14 @@ import { Util, AsyncQueue } from '@travetto/runtime';
 import { StyleUtil, Terminal, TerminalUtil } from '@travetto/terminal';
 
 import type { TestEvent } from '../../model/event.ts';
-import type { TestResult } from '../../model/test.ts';
+import type { TestResult, TestStatus } from '../../model/test.ts';
 
 import type { SuitesSummary, TestConsumerShape, TestRunState } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
 
 import { TapEmitter } from './tap.ts';
 import { CONSOLE_ENHANCER, type TestResultsEnhancer } from '../enhancer.ts';
+import type { SuiteResult } from '../../model/suite.ts';
 
 type Result = {
   key: string;
@@ -22,12 +23,7 @@ type Result = {
 @TestConsumer()
 export class TapSummaryEmitter implements TestConsumerShape {
 
-  #timings = new Map([
-    ['file', new Map<string, Result>()],
-    ['module', new Map<string, Result>()],
-    ['suite', new Map<string, Result>()],
-    ['test', new Map<string, Result>()],
-  ] as const);
+  #timings = new Map<'test' | 'module' | 'file' | 'suite', Map<string, Result>>();
 
   #terminal: Terminal;
   #results = new AsyncQueue<TestResult>();
@@ -42,6 +38,50 @@ export class TapSummaryEmitter implements TestConsumerShape {
     this.#consumer = new TapEmitter(this.#terminal, this.#enhancer);
   }
 
+  #computeTimings(suite: SuiteResult): void {
+    const [module] = suite.classId.split(/:/);
+    const [file] = suite.classId.split(/#/);
+
+    const testCount = Object.keys(suite.tests).length;
+
+    const foundModule = this.#timings
+      .getOrInsert('module', new Map<string, Result>())
+      .getOrInsert(module, { key: module, duration: 0, tests: 0 });
+
+    foundModule.duration += suite.duration;
+    foundModule.tests += testCount;
+
+    const foundFile = this.#timings
+      .getOrInsert('file', new Map<string, Result>())
+      .getOrInsert(file, { key: file, duration: 0, tests: 0 });
+
+    foundFile.duration += suite.duration;
+    foundFile.tests += testCount;
+
+    this.#timings
+      .getOrInsert('suite', new Map<string, Result>())
+      .set(suite.classId, {
+        key: suite.classId,
+        duration: suite.duration,
+        tests: testCount
+      });
+  }
+
+  #renderTimings(): void {
+    const count = +(this.#options?.count ?? 5);
+    this.#consumer.log('\n---');
+    for (const [title, results] of [...this.#timings.entries()].toSorted((a, b) => a[0].localeCompare(b[0]))) {
+      this.#consumer.log(`${this.#enhancer.suiteName(`Top ${count} slowest ${title}s`)}: `);
+      const top10 = [...results.values()].toSorted((a, b) => b.duration - a.duration).slice(0, count);
+
+      for (const result of top10) {
+        console.log(`  * ${this.#enhancer.testName(result.key)} - ${this.#enhancer.total(result.duration)}ms / ${this.#enhancer.total(result.tests)} tests`);
+      }
+      this.#consumer.log('');
+    }
+    this.#consumer.log('...');
+  }
+
   setOptions(options?: Record<string, unknown>): Promise<void> | void {
     this.#options = options;
     this.#consumer.setOptions(options);
@@ -49,22 +89,20 @@ export class TapSummaryEmitter implements TestConsumerShape {
 
   async onStart(state: TestRunState): Promise<void> {
     this.#consumer.onStart();
-
-    let failed = 0;
-    let skipped = 0;
-    let completed = 0;
+    const total: Record<TestStatus | 'count', number> = { errored: 0, failed: 0, passed: 0, skipped: 0, unknown: 0, count: 0 };
     const success = StyleUtil.getStyle({ text: '#e5e5e5', background: '#026020' }); // White on dark green
     const fail = StyleUtil.getStyle({ text: '#e5e5e5', background: '#8b0000' }); // White on dark red
     this.#progress = this.#terminal.streamToBottom(
       Util.mapAsyncIterable(
         this.#results,
         (value) => {
-          failed += (value.status === 'failed' ? 1 : 0);
-          skipped += (value.status === 'skipped' ? 1 : 0);
-          completed += (value.status !== 'skipped' ? 1 : 0);
-          return { value: `Tests %idx/%total [${failed} failed, ${skipped} skipped] -- ${value.classId}`, total: state.testCount, idx: completed };
+          total[value.status] += 1;
+          total.count += 1;
+          const statusLine = `${total.failed} failed, ${total.errored} errored, ${total.skipped} skipped`;
+          return { value: `Tests %idx/%total [${statusLine}] -- ${value.classId}`, total: state.testCount, idx: total.passed };
+
         },
-        TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: failed ? fail : success }) })
+        TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: (total.failed || total.errored) ? fail : success }) })
       ),
       { minDelay: 100 }
     );
@@ -77,40 +115,16 @@ export class TapSummaryEmitter implements TestConsumerShape {
       if (test.status === 'failed') {
         this.#consumer.onEvent(event);
       }
-      const tests = this.#timings.get('test')!;
+      const tests = this.#timings.getOrInsert('test', new Map<string, Result>());
       tests.set(`${event.test.classId}/${event.test.methodName}`, {
         key: `${event.test.classId}/${event.test.methodName}`,
         duration: test.duration,
         tests: 1
       });
     } else if (event.type === 'suite' && event.phase === 'after') {
-      const [module] = event.suite.classId.split(/:/);
-      const [file] = event.suite.classId.split(/#/);
-
-      const modules = this.#timings.get('module')!;
-      const files = this.#timings.get('file')!;
-      const suites = this.#timings.get('suite')!;
-
-      if (!modules!.has(module)) {
-        modules.set(module, { key: module, duration: 0, tests: 0 });
-      }
-
-      if (!files.has(file)) {
-        files.set(file, { key: file, duration: 0, tests: 0 });
+      if (this.#options?.timings) {
+        this.#computeTimings(event.suite);
       }
-
-      const testCount = Object.keys(event.suite.tests).length;
-
-      suites.set(event.suite.classId, {
-        key: event.suite.classId,
-        duration: event.suite.duration,
-        tests: testCount
-      });
-
-      files.get(file)!.duration += event.suite.duration;
-      files.get(file)!.tests += testCount;
-      modules.get(module)!.duration += event.suite.duration;
-      modules.get(module)!.tests += testCount;
     }
   }
 
@@ -120,21 +134,10 @@ export class TapSummaryEmitter implements TestConsumerShape {
   async onSummary(summary: SuitesSummary): Promise<void> {
     this.#results.close();
     await this.#progress;
-    await this.#consumer.onSummary?.(summary);
+    this.#consumer.onSummary?.(summary);
 
     if (this.#options?.timings) {
-      const count = +(this.#options?.count ?? 5);
-      await this.#consumer.log('\n---');
-      for (const [title, results] of [...this.#timings.entries()].toSorted((a, b) => a[0].localeCompare(b[0]))) {
-        await this.#consumer.log(`${this.#enhancer.suiteName(`Top ${count} slowest ${title}s`)}: `);
-        const top10 = [...results.values()].toSorted((a, b) => b.duration - a.duration).slice(0, count);
-
-        for (const result of top10) {
-          console.log(`  * ${this.#enhancer.testName(result.key)} - ${this.#enhancer.total(result.duration)}ms / ${this.#enhancer.total(result.tests)} tests`);
-        }
-        await this.#consumer.log('');
-      }
-      await this.#consumer.log('...');
+      this.#renderTimings();
     }
   }
 }