@travetto/test 8.0.0-alpha.0 → 8.0.0-alpha.10

package/README.md

@@ -231,7 +231,7 @@ Options:
   -c, --concurrency <number>     Number of tests to run concurrently (default: 9)
   -t, --tags <string>            Tags to target or exclude when using globs
   -o, --format-options <string>  Format options
-  -h, --help                     display help for command
+  --help                         display help for command
 ```
 
 The regexes are the patterns of tests you want to run, and all tests must be found under the `test/` folder.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/test",
-  "version": "8.0.0-alpha.0",
+  "version": "8.0.0-alpha.10",
   "type": "module",
   "description": "Declarative test framework",
   "keywords": [
@@ -28,15 +28,15 @@
     "directory": "module/test"
   },
   "dependencies": {
-    "@travetto/registry": "^8.0.0-alpha.0",
-    "@travetto/runtime": "^8.0.0-alpha.0",
-    "@travetto/terminal": "^8.0.0-alpha.0",
-    "@travetto/worker": "^8.0.0-alpha.0",
-    "yaml": "^2.8.2"
+    "@travetto/registry": "^8.0.0-alpha.10",
+    "@travetto/runtime": "^8.0.0-alpha.10",
+    "@travetto/terminal": "^8.0.0-alpha.10",
+    "@travetto/worker": "^8.0.0-alpha.10",
+    "yaml": "^2.8.3"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.0",
-    "@travetto/transformer": "^8.0.0-alpha.0"
+    "@travetto/cli": "^8.0.0-alpha.15",
+    "@travetto/transformer": "^8.0.0-alpha.5"
   },
   "peerDependenciesMeta": {
     "@travetto/transformer": {

package/src/assert/check.ts

@@ -178,17 +178,13 @@ export class AssertCheck {
   static #onError(
     positive: boolean,
     message: string | undefined,
-    error: unknown,
+    errorValue: unknown,
     missed: Error | undefined,
     shouldThrow: ThrowableError | undefined,
     assertion: CapturedAssertion
   ): void {
-    if (!(error instanceof Error)) {
-      error = new Error(`${error}`);
-    }
-    if (!(error instanceof Error)) {
-      throw error;
-    }
+    const error = errorValue instanceof Error ? errorValue : new Error(`${errorValue}`);
+
     if (positive) {
       missed = new assert.AssertionError({ message: 'Error thrown, but expected no errors' });
       missed.stack = error.stack;

package/src/assert/util.ts

@@ -1,10 +1,9 @@
 import util from 'node:util';
-import path from 'node:path';
 
-import { asFull, type Class, JSONUtil, hasFunction, Runtime, RuntimeIndex, Util } from '@travetto/runtime';
+import { JSONUtil, hasFunction, RuntimeIndex, Util } from '@travetto/runtime';
 
-import type { TestConfig, TestResult } from '../model/test.ts';
-import type { SuiteConfig, SuiteResult } from '../model/suite.ts';
+import type { Assertion, TestConfig } from '../model/test.ts';
+import type { SuiteConfig } from '../model/suite.ts';
 
 const isCleanable = hasFunction<{ toClean(): unknown }>('toClean');
 
@@ -40,93 +39,33 @@ export class AssertUtil {
    * Determine file location for a given error and the stack trace
    */
   static getPositionOfError(error: Error): { import: string, line: number } | undefined {
-    const stack = error.stack ?? new Error().stack!;
-    const frames = Util.stackTraceToParts(stack)
+    const frames = Util.stackTraceToParts(error.stack ?? new Error().stack!)
       .map(frame => {
-        const imp = (RuntimeIndex.getFromSource(frame.filename) ?? RuntimeIndex.getEntry(frame.filename))?.import;
-        return { ...frame, import: imp! };
-      })
-      .filter(frame => !!frame.import);
+        const entry = RuntimeIndex.getEntry(frame.filename);
+        return { ...frame, import: entry?.import!, line: entry?.type === 'ts' ? frame.line : 1 };
+      });
 
-    return frames[0];
+    return frames.find(frame => frame.import);
   }
 
   /**
    * Generate a suite error given a suite config, and an error
    */
-  static generateSuiteTestFailure(config: { suite: SuiteConfig, test: TestConfig, error: Error, importLocation?: string }): TestResult {
-    const { suite, test, error, importLocation } = config;
+  static generateAssertion(config: { suite: SuiteConfig, test: TestConfig, error: Error, importLocation?: string }): Assertion {
+    const { suite, test, error: errorValue, importLocation } = config;
+    const error = (errorValue.cause && errorValue.cause instanceof Error) ? errorValue.cause : errorValue;
     const testImport = importLocation ?? test.import;
     const position = this.getPositionOfError(error);
     const line = position?.line ?? (testImport === suite.import ? suite.lineStart : 1);
-    const testResult: TestResult = {
-      ...suite.tests[test.methodName],
-      suiteLineStart: suite.lineStart,
-      status: 'errored',
+    return {
+      import: position?.import ?? testImport,
+      methodName: test.methodName,
+      classId: suite.classId,
+      operator: 'throw',
       error,
-      duration: 0,
-      durationTotal: 0,
-      output: [],
-      assertions: [{
-        import: position?.import ?? testImport,
-        methodName: test.methodName,
-        classId: suite.classId,
-        operator: 'throw',
-        error,
-        line,
-        message: error.message.split(/\n/)[0],
-        text: test.methodName
-      }],
+      line,
+      message: error.message.split(/\n/)[0],
+      text: test.methodName
     };
-
-    return testResult;
-  }
-
-  /**
-   * Generate suite failure
-   */
-  static generateSuiteTestFailures(suite: SuiteConfig, error: Error): TestResult[] {
-    const finalError = error.cause instanceof Error ? error.cause : error;
-    return Object.values(suite.tests).map(test => this.generateSuiteTestFailure({ suite, test, error: finalError }));
-  }
-
-  /**
-   * Define import failure as a TestResult
-   */
-  static gernerateImportFailure(importLocation: string, error: Error): { result: TestResult, test: TestConfig, suite: SuiteResult & SuiteConfig } {
-    const name = path.basename(importLocation);
-    const classId = `${RuntimeIndex.getFromImport(importLocation)?.id}#${name}`;
-    const suite = asFull<SuiteConfig & SuiteResult>({
-      class: asFull<Class>({ name }), classId, duration: 0, lineStart: 1, lineEnd: 1, import: importLocation
-    });
-    error.message = error.message.replaceAll(Runtime.mainSourcePath, '.');
-    const result = this.generateSuiteTestFailure({
-      suite,
-      test: {
-        methodName: 'require',
-        classId,
-        import: importLocation,
-        class: suite.class,
-        lineBodyStart: 1,
-        lineStart: 1,
-        lineEnd: 1,
-        skip: false
-      },
-      error
-    });
-    const test: TestConfig = {
-      methodName: 'import',
-      classId,
-      import: importLocation,
-      declarationImport: importLocation,
-      lineStart: 0,
-      lineEnd: 0,
-      lineBodyStart: 0,
-      tags: [],
-      description: 'Import Failure',
-      skip: false,
-      class: undefined!
-    };
-    return { result, test, suite };
   }
 }

package/src/consumer/types/cumulative.ts

@@ -1,7 +1,7 @@
 import type { TestConsumerShape } from '../types.ts';
 import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConfig, TestDiffSource, TestResult } from '../../model/test.ts';
-import type { Counts, SuiteConfig, SuiteResult } from '../../model/suite.ts';
+import type { SuiteConfig, SuiteResult } from '../../model/suite.ts';
 import { DelegatingConsumer } from './delegating.ts';
 import type { SuiteCore } from '../../model/common.ts';
 import { TestModelUtil } from '../../model/util.ts';
@@ -9,7 +9,7 @@ import { TestModelUtil } from '../../model/util.ts';
 type ClassId = string;
 type ImportName = string;
 
-type CumulativeTestResult = Pick<TestResult, 'sourceHash' | 'status' | 'duration'>;
+type CumulativeTestResult = Pick<TestResult, 'sourceHash' | 'status' | 'duration' | 'selfDuration'>;
 type CumulativeSuiteResult = Pick<SuiteCore, 'import' | 'classId' | 'sourceHash'> & {
   tests: Record<string, CumulativeTestResult>;
 };
@@ -37,7 +37,7 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
 
   onTestBefore(config: TestConfig): TestConfig {
     const suite = this.getSuite(config);
-    suite.tests[config.methodName] = { sourceHash: config.sourceHash, status: 'unknown', duration: 0 };
+    suite.tests[config.methodName] = { sourceHash: config.sourceHash, status: 'unknown', duration: 0, selfDuration: 0 };
     return config;
   }
 
@@ -56,21 +56,9 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
   onSuiteAfter(result: SuiteResult): SuiteResult {
     // Reset counts
     const suite = this.getSuite(result);
-    const totals: Counts & { duration: number } = {
-      passed: 0,
-      failed: 0,
-      skipped: 0,
-      errored: 0,
-      unknown: 0,
-      total: 0,
-      duration: 0
-    };
-    for (const test of Object.values(suite.tests)) {
-      totals[test.status] += 1;
-      totals.total += 1;
-      totals.duration += test.duration ?? 0;
-    }
-    return { ...result, ...totals, status: TestModelUtil.countsToTestStatus(totals) };
+    const results = TestModelUtil.buildSummary();
+    TestModelUtil.countTestResult(results, Object.values(suite.tests));
+    return { ...result, ...results, status: TestModelUtil.computeTestStatus(results) };
   }
 
   removeTest(importName: string, classId?: string, methodName?: string): void {

package/src/consumer/types/delegating.ts

@@ -20,6 +20,12 @@ export abstract class DelegatingConsumer implements TestConsumerShape {
     }
   }
 
+  async onTestRunState(state: TestRunState): Promise<void> {
+    for (const consumer of this.#consumers) {
+      await consumer.onTestRunState?.(state);
+    }
+  }
+
   onRemoveEvent(event: TestRemoveEvent): void {
     let result = event;
     if (this.transformRemove) {

package/src/consumer/types/runnable.ts

@@ -22,6 +22,7 @@ export class RunnableTestConsumer extends DelegatingConsumer {
 
   async summarizeAsBoolean(): Promise<boolean> {
     await this.summarize(this.#results?.summary);
-    return (this.#results?.summary.failed ?? 0) <= 0;
+    return (this.#results?.summary.failed ?? 0) <= 0 &&
+      (this.#results?.summary.errored ?? 0) <= 0;
   }
 }

package/src/consumer/types/summarizer.ts

@@ -1,6 +1,7 @@
 import type { SuiteResult } from '../../model/suite.ts';
 import type { TestEvent } from '../../model/event.ts';
 import type { SuitesSummary, TestConsumerShape } from '../types.ts';
+import { TestModelUtil } from '../../model/util.ts';
 
 /**
  * Test Result Collector, combines all results into a single Suite Result
@@ -8,26 +9,13 @@ import type { SuitesSummary, TestConsumerShape } from '../types.ts';
 export class TestResultsSummarizer implements TestConsumerShape {
 
   summary: SuitesSummary = {
-    passed: 0,
-    failed: 0,
-    errored: 0,
-    skipped: 0,
-    unknown: 0,
-    total: 0,
-    duration: 0,
-    suites: [],
-    errors: []
+    ...TestModelUtil.buildSummary(),
+    suites: []
   };
 
   #merge(result: SuiteResult): void {
+    TestModelUtil.countTestResult(this.summary, Object.values(result.tests));
     this.summary.suites.push(result);
-    this.summary.failed += result.failed;
-    this.summary.errored += result.errored;
-    this.summary.passed += result.passed;
-    this.summary.unknown += result.unknown;
-    this.summary.skipped += result.skipped;
-    this.summary.duration += result.duration;
-    this.summary.total += result.total;
   }
 
   /**

package/src/consumer/types/tap-summary.ts

@@ -2,7 +2,7 @@ import { Util, AsyncQueue } from '@travetto/runtime';
 import { StyleUtil, Terminal, TerminalUtil } from '@travetto/terminal';
 
 import type { TestEvent } from '../../model/event.ts';
-import type { TestResult, TestStatus } from '../../model/test.ts';
+import type { TestResult } from '../../model/test.ts';
 
 import type { SuitesSummary, TestConsumerShape, TestRunState } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
@@ -10,6 +10,7 @@ import { TestConsumer } from '../decorator.ts';
 import { TapEmitter } from './tap.ts';
 import { CONSOLE_ENHANCER, type TestResultsEnhancer } from '../enhancer.ts';
 import type { SuiteResult } from '../../model/suite.ts';
+import { TestModelUtil } from '../../model/util.ts';
 
 type Result = {
   key: string;
@@ -31,6 +32,7 @@ export class TapSummaryEmitter implements TestConsumerShape {
   #consumer: TapEmitter;
   #enhancer: TestResultsEnhancer;
   #options?: Record<string, unknown>;
+  #state: TestRunState = {};
 
   constructor(terminal: Terminal = new Terminal(process.stderr)) {
     this.#terminal = terminal;
@@ -87,20 +89,24 @@ export class TapSummaryEmitter implements TestConsumerShape {
     this.#consumer.setOptions(options);
   }
 
+  onTestRunState(state: TestRunState): void {
+    Object.assign(this.#state, state);
+  }
+
   async onStart(state: TestRunState): Promise<void> {
     this.#consumer.onStart();
-    const total: Record<TestStatus | 'count', number> = { errored: 0, failed: 0, passed: 0, skipped: 0, unknown: 0, count: 0 };
+    this.onTestRunState(state);
+
+    const total = TestModelUtil.buildSummary();
     const success = StyleUtil.getStyle({ text: '#e5e5e5', background: '#026020' }); // White on dark green
     const fail = StyleUtil.getStyle({ text: '#e5e5e5', background: '#8b0000' }); // White on dark red
     this.#progress = this.#terminal.streamToBottom(
       Util.mapAsyncIterable(
         this.#results,
         (value) => {
-          total[value.status] += 1;
-          total.count += 1;
+          TestModelUtil.countTestResult(total, [value]);
           const statusLine = `${total.failed} failed, ${total.errored} errored, ${total.skipped} skipped`;
-          return { value: `Tests %idx/%total [${statusLine}] -- ${value.classId}`, total: state.testCount, idx: total.count };
-
+          return { value: `Tests %idx/%total [${statusLine}] -- ${value.classId}`, total: this.#state.testCount, idx: total.passed };
         },
         TerminalUtil.progressBarUpdater(this.#terminal, { style: () => ({ complete: (total.failed || total.errored) ? fail : success }) })
       ),
@@ -112,7 +118,7 @@ export class TapSummaryEmitter implements TestConsumerShape {
     if (event.type === 'test' && event.phase === 'after') {
       const { test } = event;
       this.#results.add(test);
-      if (test.status === 'failed') {
+      if (test.status !== 'passed' && test.status !== 'skipped') {
         this.#consumer.onEvent(event);
       }
       const tests = this.#timings.getOrInsert('test', new Map<string, Result>());

package/src/consumer/types/tap.ts

@@ -2,12 +2,15 @@ import path from 'node:path';
 import { stringify } from 'yaml';
 
 import { Terminal, StyleUtil } from '@travetto/terminal';
-import { TimeUtil, RuntimeIndex, hasToJSON, JSONUtil } from '@travetto/runtime';
+import { TimeUtil, RuntimeIndex } from '@travetto/runtime';
 
 import type { TestEvent } from '../../model/event.ts';
 import type { SuitesSummary, TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
 import { type TestResultsEnhancer, CONSOLE_ENHANCER } from '../enhancer.ts';
+import { TestConsumerUtil } from './util.ts';
+
+const SPACE = ' ';
 
 /**
  * TAP Format consumer
@@ -17,8 +20,8 @@ export class TapEmitter implements TestConsumerShape {
   #count = 0;
   #enhancer: TestResultsEnhancer;
   #terminal: Terminal;
-  #start: number;
   #options?: Record<string, unknown>;
+  #start: number = 0;
 
   constructor(
     terminal = new Terminal(),
@@ -54,24 +57,6 @@ export class TapEmitter implements TestConsumerShape {
     this.log(`---\n${this.#enhancer.objectInspect(body)}\n...`);
   }
 
-  /**
-   * Error to string
-   * @param error
-   */
-  errorToString(error?: Error): string | undefined {
-    if (error && error.name !== 'AssertionError') {
-      if (error instanceof Error) {
-        let out = JSONUtil.toUTF8(hasToJSON(error) ? error.toJSON() : error, { indent: 2 });
-        if (this.#options?.verbose && error.stack) {
-          out = `${out}\n${error.stack}`;
-        }
-        return out;
-      } else {
-        return `${error}`;
-      }
-    }
-  }
-
   /**
    * Listen for each event
    */
@@ -125,9 +110,10 @@ export class TapEmitter implements TestConsumerShape {
       // Track test result
       let status = `${this.#enhancer.testNumber(++this.#count)} `;
       switch (test.status) {
+        case 'passed': `${this.#enhancer.success('ok')} ${status}`; break;
         case 'skipped': status += ' # SKIP'; break;
-        case 'failed': status = `${this.#enhancer.failure('not ok')} ${status}`; break;
-        default: status = `${this.#enhancer.success('ok')} ${status}`;
+        case 'unknown': break;
+        default: status = `${this.#enhancer.failure('not ok')} ${status}`; break;
       }
       status += header;
 
@@ -138,9 +124,9 @@ export class TapEmitter implements TestConsumerShape {
         case 'errored':
         case 'failed': {
           if (test.error) {
-            const msg = this.errorToString(test.error);
-            if (msg) {
-              this.logMeta({ error: msg });
+            const message = TestConsumerUtil.errorToString(test.error, !!this.#options?.verbose);
+            if (message) {
+              this.logMeta({ error: message });
             }
           }
           break;
@@ -168,28 +154,22 @@ export class TapEmitter implements TestConsumerShape {
   onSummary(summary: SuitesSummary): void {
     this.log(`${this.#enhancer.testNumber(1)}..${this.#enhancer.testNumber(summary.total)}`);
 
-    if (summary.errors.length) {
-      this.log('---\n');
-      for (const error of summary.errors) {
-        const msg = this.errorToString(error);
-        if (msg) {
-          this.log(this.#enhancer.failure(msg));
-        }
-      }
-    }
-
     const allPassed = !summary.failed && !summary.errored;
 
     this.log([
-      this.#enhancer[allPassed ? 'success' : 'failure']('Results'),
-      `${this.#enhancer.total(summary.passed)}/${this.#enhancer.total(summary.total)},`,
-      allPassed ? 'failed' : this.#enhancer.failure('failed'),
-      `${this.#enhancer.total(summary.failed)}`,
-      allPassed ? 'errored' : this.#enhancer.failure('errored'),
-      `${this.#enhancer.total(summary.errored)}`,
-      'skipped',
-      this.#enhancer.total(summary.skipped),
-      `# (Total Test Time: ${TimeUtil.asClock(summary.duration)}, Total Run Time: ${TimeUtil.asClock(Date.now() - this.#start)})`
-    ].join(' '));
+      this.#enhancer[allPassed ? 'success' : 'failure']('Results'), SPACE,
+      `${this.#enhancer.total(summary.passed)}/${this.#enhancer.total(summary.total)},`, SPACE,
+      allPassed ? 'failed' : this.#enhancer.failure('failed'), SPACE,
+      `${this.#enhancer.total(summary.failed)}`, SPACE,
+      allPassed ? 'errored' : this.#enhancer.failure('errored'), SPACE,
+      `${this.#enhancer.total(summary.errored)}`, SPACE,
+      'skipped', SPACE,
+      this.#enhancer.total(summary.skipped), SPACE,
+      '#', SPACE, '(Timings:', SPACE,
+      'Self=', TimeUtil.asClock(summary.selfDuration), ',', SPACE,
+      'Total=', TimeUtil.asClock(summary.duration), ',', SPACE,
+      'Clock=', TimeUtil.asClock(Date.now() - this.#start),
+      ')',
+    ].join(''));
   }
 }

package/src/consumer/types/util.ts

@@ -0,0 +1,28 @@
+import util from 'node:util';
+import { AssertionError } from 'node:assert';
+
+import { TypedObject } from '@travetto/runtime';
+
+export class TestConsumerUtil {
+  /**
+   * Convert error to string
+   */
+  static errorToString(error?: Error, verbose?: boolean): string | undefined {
+    if (error instanceof AssertionError) {
+      return;
+    } else if (error instanceof Error) {
+      const stack = error.stack ?
+        error.stack.split(/\n/).slice(0, verbose ? -1 : 5).join('\n') :
+        error.message;
+      const subObject: Record<string, unknown> = {};
+      for (const key of TypedObject.keys(error)) {
+        if (key !== 'stack' && key !== 'message' && key !== 'name') {
+          subObject[key] = error[key];
+        }
+      }
+      return `${stack}${Object.keys(subObject).length ? `\n${util.inspect(subObject)}` : ''}`;
+    } else {
+      return `${error}`;
+    }
+  }
+}

package/src/consumer/types/xunit.ts

@@ -7,6 +7,7 @@ import { RuntimeIndex } from '@travetto/runtime';
 import type { TestEvent } from '../../model/event.ts';
 import type { SuitesSummary, TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
+import { TestConsumerUtil } from './util.ts';
 
 /**
  * Xunit consumer, compatible with JUnit formatters
@@ -59,9 +60,9 @@ export class XunitEmitter implements TestConsumerShape {
       let body = '';
 
       if (test.error) {
-        const assertion = test.assertions.find(item => !!item.error)!;
+        const errorMessage = TestConsumerUtil.errorToString(test.error);
         const node = test.status === 'failed' ? 'failure' : 'error';
-        body = `<${node} type="${assertion.text}" message="${encodeURIComponent(assertion.message!)}"><![CDATA[${assertion.error!.stack}]]></${node}>`;
+        body = `<${node} type="${test.error.constructor.name}" message="${encodeURIComponent(test.error.message)}"><![CDATA[${errorMessage}]]></${node}>`;
       }
 
       const groupedByLevel: Record<string, string[]> = {};

package/src/consumer/types.ts

@@ -1,24 +1,16 @@
 import type { Class } from '@travetto/runtime';
 
 import type { TestEvent, TestRemoveEvent } from '../model/event.ts';
-import type { Counts, SuiteResult } from '../model/suite.ts';
+import type { ResultsSummary, SuiteResult } from '../model/suite.ts';
 
 /**
  * All suite results
  */
-export interface SuitesSummary extends Counts {
+export interface SuitesSummary extends ResultsSummary {
   /**
    * List of all suites
    */
   suites: SuiteResult[];
-  /**
-   * List of all errors
-   */
-  errors: Error[];
-  /**
-   * Total duration
-   */
-  duration: number;
 }
 
 export type TestRunState = {
@@ -44,6 +36,10 @@ export interface TestConsumerShape {
    * Set options
    */
   setOptions?(options?: Record<string, unknown>): Promise<void> | void;
+  /**
+   * Can directly update the known test run state as needed
+   */
+  onTestRunState?(state: TestRunState): Promise<void> | void;
   /**
    * Listen for start of the test run
    */

package/src/decorator/suite.ts

@@ -1,4 +1,4 @@
-import { castTo, type Class, type ClassInstance, describeFunction, getClass } from '@travetto/runtime';
+import { castTo, type Class, type ClassInstance, getClass } from '@travetto/runtime';
 
 import type { SuiteConfig } from '../model/suite.ts';
 import { SuiteRegistryIndex } from '../registry/registry-index.ts';
@@ -15,11 +15,9 @@ export function Suite(...rest: Partial<SuiteConfig>[]): ClassDecorator;
 export function Suite(description: string, ...rest: Partial<SuiteConfig>[]): ClassDecorator;
 export function Suite(description?: string | Partial<SuiteConfig>, ...rest: Partial<SuiteConfig>[]): ClassDecorator {
   const decorator = (cls: Class): typeof cls => {
-    const isAbstract = describeFunction(cls).abstract;
     SuiteRegistryIndex.getForRegister(cls).register(
       ...(typeof description !== 'string' && description ? [description] : []),
       ...rest,
-      ...isAbstract ? [{ skip: true }] : [],
       ...(typeof description === 'string' ? [{ description }] : []),
     );
     return cls;