@travetto/test 7.0.0-rc.2 → 7.0.0-rc.3

package/README.md

@@ -231,8 +231,7 @@ Usage: test [options] [first:string] [globs...:string]
 Options:
   -f, --format <string>          Output format for test results (default: "tap")
   -c, --concurrency <number>     Number of tests to run concurrently (default: 9)
-  -m, --mode <single|standard>   Test run mode (default: "standard")
-  -t, --tags <string>            Tags to target or exclude
+  -t, --tags <string>            Tags to target or exclude when using globs
   -o, --format-options <string>  Format options
   -h, --help                     display help for command
 ```

package/__index__.ts

@@ -4,6 +4,7 @@ export * from './src/decorator/test.ts';
 export * from './src/model/suite.ts';
 export * from './src/model/test.ts';
 export * from './src/model/event.ts';
+export * from './src/model/util.ts';
 export * from './src/registry/registry-index.ts';
 export * from './src/registry/registry-adapter.ts';
 export * from './src/fixture.ts';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/test",
-  "version": "7.0.0-rc.2",
+  "version": "7.0.0-rc.3",
   "description": "Declarative test framework",
   "keywords": [
     "unit-testing",
@@ -27,15 +27,15 @@
     "directory": "module/test"
   },
   "dependencies": {
-    "@travetto/registry": "^7.0.0-rc.2",
-    "@travetto/runtime": "^7.0.0-rc.2",
-    "@travetto/terminal": "^7.0.0-rc.2",
-    "@travetto/worker": "^7.0.0-rc.2",
+    "@travetto/registry": "^7.0.0-rc.3",
+    "@travetto/runtime": "^7.0.0-rc.3",
+    "@travetto/terminal": "^7.0.0-rc.3",
+    "@travetto/worker": "^7.0.0-rc.3",
     "yaml": "^2.8.1"
   },
   "peerDependencies": {
-    "@travetto/cli": "^7.0.0-rc.2",
-    "@travetto/transformer": "^7.0.0-rc.2"
+    "@travetto/cli": "^7.0.0-rc.3",
+    "@travetto/transformer": "^7.0.0-rc.3"
   },
   "peerDependenciesMeta": {
     "@travetto/transformer": {

package/src/assert/util.ts

@@ -96,7 +96,7 @@ export class AssertUtil {
 
     const msg = error.message.split(/\n/)[0];
 
-    const core = { import: imp, classId: suite.classId, methodName };
+    const core = { import: imp, classId: suite.classId, methodName, sourceHash: suite.sourceHash };
     const coreAll = { ...core, description: msg, lineStart: line, lineEnd: line, lineBodyStart: line };
 
     const assert: Assertion = {

package/src/communication.ts

@@ -0,0 +1,66 @@
+import { AppError, hasToJSON, JSONUtil } from '@travetto/runtime';
+
+/**
+ * Tools for communication serialization/deserialization especially with errors
+ */
+export class CommunicationUtil {
+
+  /**
+   * Serialize to JSON
+   */
+  static serialize<T>(out: T): string {
+    return JSON.stringify(out, function (key, value) {
+      const objectValue = this[key];
+      if (objectValue && objectValue instanceof Error) {
+        return {
+          $: true,
+          ...hasToJSON(objectValue) ? objectValue.toJSON() : objectValue,
+          name: objectValue.name,
+          message: objectValue.message,
+          stack: objectValue.stack,
+        };
+      } else if (typeof value === 'bigint') {
+        return `${value.toString()}$n`;
+      } else {
+        return value;
+      }
+    });
+  }
+
+  /**
+   * Serialize to a standard object, instead of a string
+   */
+  static serializeToObject<R = Record<string, unknown>, T = unknown>(out: T): R {
+    return JSONUtil.parseSafe(this.serialize(out));
+  }
+
+  /**
+   * Deserialize from JSON
+   */
+  static deserialize<T = unknown>(input: string): T {
+    return JSONUtil.parseSafe(input, function (key, value) {
+      if (value && typeof value === 'object' && '$' in value) {
+        const error = AppError.fromJSON(value) ?? new Error();
+        if (!(error instanceof AppError)) {
+          const { $: _, ...rest } = value;
+          Object.assign(error, rest);
+        }
+        error.message = value.message;
+        error.stack = value.stack;
+        error.name = value.name;
+        return error;
+      } else if (typeof value === 'string' && /^\d+[$]n$/.test(value)) {
+        return BigInt(value.split('$')[0]);
+      } else {
+        return value;
+      }
+    });
+  }
+
+  /**
+   * Deserialize from a standard object, instead of a string
+   */
+  static deserializeFromObject<R = unknown, T = R>(input: T): R {
+    return this.deserialize<R>(JSON.stringify(input));
+  }
+}

package/src/consumer/registry-index.ts

@@ -2,7 +2,7 @@ import { RuntimeIndex, type Class } from '@travetto/runtime';
 import { Registry, RegistryIndex, RegistryIndexStore } from '@travetto/registry';
 
 import type { TestConsumerShape } from './types.ts';
-import type { RunState } from '../execute/types.ts';
+import type { TestConsumerConfig } from '../execute/types.ts';
 import { TestConsumerRegistryAdapter } from './registry-adapter.ts';
 
 /**
@@ -25,15 +25,17 @@ export class TestConsumerRegistryIndex implements RegistryIndex {
 
   /**
    * Get a consumer instance that supports summarization
-   * @param consumer The consumer identifier or the actual consumer
+   * @param consumerConfig The consumer configuration
    */
-  static getInstance(state: Pick<RunState, 'consumer' | 'consumerOptions'>): Promise<TestConsumerShape> {
-    return this.#instance.getInstance(state);
+  static getInstance(consumerConfig: TestConsumerConfig): Promise<TestConsumerShape> {
+    return this.#instance.getInstance(consumerConfig);
   }
 
   #initialized: Promise<void>;
   store = new RegistryIndexStore(TestConsumerRegistryAdapter);
 
+  /** @private */ constructor(source: unknown) { Registry.validateConstructor(source); }
+
   /**
    * Manual initialization when running outside of the bootstrap process
    */
@@ -50,8 +52,6 @@ export class TestConsumerRegistryIndex implements RegistryIndex {
     }
   }
 
-  process(): void { }
-
   /**
    * Get types
    */
@@ -69,7 +69,7 @@ export class TestConsumerRegistryIndex implements RegistryIndex {
    * Get a consumer instance that supports summarization
    * @param consumer The consumer identifier or the actual consumer
    */
-  async getInstance(state: Pick<RunState, 'consumer' | 'consumerOptions'>): Promise<TestConsumerShape> {
+  async getInstance(state: Pick<TestConsumerConfig, 'consumer' | 'consumerOptions'>): Promise<TestConsumerShape> {
     await (this.#initialized ??= this.#init());
     for (const cls of this.store.getClasses()) {
       const adapter = this.store.get(cls);

package/src/consumer/types/cumulative.ts

@@ -1,13 +1,18 @@
-import { existsSync } from 'node:fs';
-
-import { type Class, RuntimeIndex } from '@travetto/runtime';
-
 import type { TestConsumerShape } from '../types.ts';
-import type { TestEvent } from '../../model/event.ts';
-import type { TestResult } from '../../model/test.ts';
-import type { SuiteResult } from '../../model/suite.ts';
+import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
+import type { TestConfig, TestDiffSource, TestResult } from '../../model/test.ts';
+import type { Counts, SuiteConfig, SuiteResult } from '../../model/suite.ts';
 import { DelegatingConsumer } from './delegating.ts';
-import { SuiteRegistryIndex } from '../../registry/registry-index.ts';
+import { SuiteCore } from '../../model/common.ts';
+import { TestModelUtil } from '../../model/util.ts';
+
+type ClassId = string;
+type ImportName = string;
+
+type CumulativeTestResult = Pick<TestResult, 'sourceHash' | 'status' | 'duration'>;
+type CumulativeSuiteResult = Pick<SuiteCore, 'import' | 'classId' | 'sourceHash'> & {
+  tests: Record<string, CumulativeTestResult>;
+};
 
 /**
  * Cumulative Summary consumer
@@ -16,77 +21,101 @@ export class CumulativeSummaryConsumer extends DelegatingConsumer {
   /**
    * Total state of all tests run so far
    */
-  #state: Record<string, Record<string, TestResult['status']>> = {};
+  #state: Record<ImportName, Record<ClassId, CumulativeSuiteResult>> = {};
 
   constructor(target: TestConsumerShape) {
     super([target]);
   }
 
-  /**
-   * Summarize a given test suite using the new result and the historical
-   * state
-   */
-  summarizeSuite(test: TestResult): SuiteResult {
-    // Was only loading to verify existence (TODO: double-check)
-    if (existsSync(RuntimeIndex.getFromImport(test.import)!.sourceFile)) {
-      (this.#state[test.classId] ??= {})[test.methodName] = test.status;
-      const SuiteCls = SuiteRegistryIndex.getClasses().find(cls => cls.Ⲑid === test.classId);
-      return SuiteCls ? this.computeTotal(SuiteCls) : this.removeClass(test.classId);
-    } else {
-      return this.removeClass(test.classId);
-    }
+  getSuite(core: Pick<SuiteCore, 'import' | 'classId'>): CumulativeSuiteResult {
+    return this.#state[core.import]?.[core.classId];
   }
 
-  /**
-   * Remove a class
-   */
-  removeClass(clsId: string): SuiteResult {
-    this.#state[clsId] = {};
-    return {
-      classId: clsId, passed: 0, failed: 0, skipped: 0, total: 0, tests: [], duration: 0, import: '', lineStart: 0, lineEnd: 0
-    };
+  getOrCreateSuite({ tests: _, ...core }: SuiteConfig | SuiteResult): CumulativeSuiteResult {
+    return (this.#state[core.import] ??= {})[core.classId] ??= { ...core, tests: {} };
   }
 
-  /**
-   * Compute totals
-   */
-  computeTotal(cls: Class): SuiteResult {
-    const suite = SuiteRegistryIndex.getConfig(cls);
-    const total = Object.values(suite.tests).reduce((map, config) => {
-      const status = this.#state[config.classId][config.methodName] ?? 'unknown';
-      map[status] += 1;
-      return map;
-    }, { skipped: 0, passed: 0, failed: 0, unknown: 0 });
+  onTestBefore(config: TestConfig): TestConfig {
+    const suite = this.getSuite(config);
+    suite.tests[config.methodName] = { sourceHash: config.sourceHash, status: 'unknown', duration: 0 };
+    return config;
+  }
+
+  onTestAfter(result: TestResult): TestResult {
+    const test = this.getSuite(result).tests[result.methodName];
+    Object.assign(test, { status: result.status, duration: result.duration });
+    return result;
+  }
 
-    return {
-      classId: suite.classId,
-      passed: total.passed,
-      failed: total.failed,
-      skipped: total.skipped,
-      import: suite.import,
-      lineStart: suite.lineStart,
-      lineEnd: suite.lineEnd,
-      total: total.failed + total.passed,
-      tests: [],
-      duration: 0
-    };
+  onSuiteBefore(config: SuiteConfig): SuiteConfig {
+    const suite = this.getOrCreateSuite(config);
+    suite.sourceHash = config.sourceHash;
+    return config;
+  }
+
+  onSuiteAfter(result: SuiteResult): SuiteResult {
+    // Reset counts
+    const suite = this.getSuite(result);
+    const totals: Counts & { duration: number } = { passed: 0, failed: 0, skipped: 0, unknown: 0, total: 0, duration: 0 };
+    for (const test of Object.values(suite.tests)) {
+      totals[test.status] += 1;
+      totals.total += 1;
+      totals.duration += test.duration ?? 0;
+    }
+    return { ...result, ...totals, status: TestModelUtil.countsToTestStatus(totals) };
+  }
+
+  removeTest(importName: string, classId?: string, methodName?: string): void {
+    if (methodName && classId && importName) {
+      delete this.getSuite({ import: importName, classId }).tests[methodName];
+    } else if (classId && importName) {
+      delete this.#state[importName][classId];
+    } else if (importName) {
+      delete this.#state[importName];
+    }
+  }
+
+  transformRemove(event: TestRemoveEvent): TestRemoveEvent {
+    this.removeTest(event.import, event.classId, event.methodName);
+    return event;
   }
 
   /**
-   * Listen for event, process the full event, and if the event is an after test,
-   * send a full suite summary
+   * Handle cumulative events, and emit a summarized summary
    */
-  onEventDone(event: TestEvent): void {
+  transform(event: TestEvent): TestEvent | undefined {
     try {
-      if (event.type === 'test' && event.phase === 'after') {
-        this.onEvent({
-          type: 'suite',
-          phase: 'after',
-          suite: this.summarizeSuite(event.test),
-        });
+      if (event.type === 'suite') {
+        if (event.phase === 'before') {
+          return { ...event, suite: this.onSuiteBefore(event.suite) };
+        } else if (event.phase === 'after') {
+          return { ...event, suite: this.onSuiteAfter(event.suite) };
+        }
+      } else if (event.type === 'test') {
+        if (event.phase === 'before') {
+          return { ...event, test: this.onTestBefore(event.test) };
+        } else if (event.phase === 'after') {
+          return { ...event, test: this.onTestAfter(event.test) };
+        }
       }
+      return event;
     } catch (error) {
       console.warn('Summarization Error', { error });
     }
   }
+
+  /**
+   * Produce diff source for import file
+   */
+  produceDiffSource(importName: string): TestDiffSource {
+    const output: TestDiffSource = {};
+    for (const [clsId, suite] of Object.entries(this.#state[importName] || {})) {
+      const methods: TestDiffSource[string]['methods'] = {};
+      for (const [methodName, test] of Object.entries(suite.tests)) {
+        methods[methodName] = test.sourceHash!;
+      }
+      output[clsId] = { sourceHash: suite.sourceHash!, methods };
+    }
+    return output;
+  }
 }

package/src/consumer/types/delegating.ts

@@ -1,13 +1,11 @@
 import type { SuitesSummary, TestConsumerShape, TestRunState } from '../types.ts';
-import type { TestEvent } from '../../model/event.ts';
+import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 
 /**
  * Delegating event consumer
  */
 export abstract class DelegatingConsumer implements TestConsumerShape {
   #consumers: TestConsumerShape[];
-  #transformer?: (event: TestEvent) => typeof event;
-  #filter?: (event: TestEvent) => boolean;
 
   constructor(consumers: TestConsumerShape[]) {
     this.#consumers = consumers;
@@ -16,34 +14,38 @@ export abstract class DelegatingConsumer implements TestConsumerShape {
     }
   }
 
-  withTransformer(transformer: (event: TestEvent) => typeof event): this {
-    this.#transformer = transformer;
-    return this;
-  }
-
-  withFilter(filter: (event: TestEvent) => boolean): this {
-    this.#filter = filter;
-    return this;
-  }
-
   async onStart(state: TestRunState): Promise<void> {
     for (const consumer of this.#consumers) {
       await consumer.onStart?.(state);
     }
   }
 
-  onEvent(event: TestEvent): void {
-    if (this.#transformer) {
-      event = this.#transformer(event);
+  onRemoveEvent(event: TestRemoveEvent): void {
+    let result = event;
+    if (this.transformRemove) {
+      result = this.transformRemove(event) ?? event;
     }
-    if (this.#filter?.(event) === false) {
-      return;
+    if (result) {
+      for (const consumer of this.#consumers) {
+        consumer.onRemoveEvent?.(result);
+      }
     }
+  }
+
+  delegateEvent(event: TestEvent): void {
     for (const consumer of this.#consumers) {
       consumer.onEvent(event);
     }
+  }
 
-    this.onEventDone?.(event);
+  onEvent(event: TestEvent): void {
+    let result = event;
+    if (this.transform) {
+      result = this.transform(event) ?? event;
+    }
+    if (result) {
+      this.delegateEvent(result);
+    }
   }
 
   async summarize(summary?: SuitesSummary): Promise<void> {
@@ -54,5 +56,6 @@ export abstract class DelegatingConsumer implements TestConsumerShape {
     }
   }
 
-  onEventDone?(event: TestEvent): void;
+  transform?(event: TestEvent): TestEvent | undefined;
+  transformRemove?(event: TestRemoveEvent): TestRemoveEvent | undefined;
 }

package/src/consumer/types/event.ts

@@ -1,10 +1,9 @@
 import { Writable } from 'node:stream';
 
-import { Util } from '@travetto/runtime';
-
-import type { TestEvent } from '../../model/event.ts';
+import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
+import { CommunicationUtil } from '../../communication.ts';
 
 /**
  * Streams all test events a JSON payload, in an nd-json format
@@ -17,7 +16,15 @@ export class EventStreamer implements TestConsumerShape {
     this.#stream = stream;
   }
 
+  sendPayload(payload: unknown): void {
+    this.#stream.write(`${CommunicationUtil.serialize(payload)}\n`);
+  }
+
   onEvent(event: TestEvent): void {
-    this.#stream.write(`${Util.serializeToJSON(event)}\n`);
+    this.sendPayload(event);
+  }
+
+  onRemoveEvent(event: TestRemoveEvent): void {
+    this.sendPayload(event);
   }
 }

package/src/consumer/types/exec.ts

@@ -1,16 +1,25 @@
 import { IpcChannel } from '@travetto/worker';
-import { Util } from '@travetto/runtime';
 
-import type { TestEvent } from '../../model/event.ts';
+import type { TestEvent, TestRemoveEvent } from '../../model/event.ts';
 import type { TestConsumerShape } from '../types.ts';
 import { TestConsumer } from '../decorator.ts';
+import { CommunicationUtil } from '../../communication.ts';
 
 /**
  * Triggers each event as an IPC command to a parent process
  */
 @TestConsumer()
 export class ExecutionEmitter extends IpcChannel<TestEvent> implements TestConsumerShape {
+
+  sendPayload(payload: unknown & { type: string }): void {
+    this.send(payload.type, CommunicationUtil.serializeToObject(payload));
+  }
+
   onEvent(event: TestEvent): void {
-    this.send(event.type, JSON.parse(Util.serializeToJSON(event)));
+    this.sendPayload(event);
+  }
+
+  onRemoveEvent(event: TestRemoveEvent): void {
+    this.sendPayload(event);
   }
 }

package/src/consumer/types/runnable.ts

@@ -15,8 +15,9 @@ export class RunnableTestConsumer extends DelegatingConsumer {
     this.#results = consumers.find(consumer => !!consumer.onSummary) ? new TestResultsSummarizer() : undefined;
   }
 
-  onEventDone(event: TestEvent): void {
+  transform(event: TestEvent): TestEvent | undefined {
     this.#results?.onEvent(event);
+    return event;
   }
 
   async summarizeAsBoolean(): Promise<boolean> {

package/src/consumer/types/summarizer.ts

@@ -11,6 +11,7 @@ export class TestResultsSummarizer implements TestConsumerShape {
     passed: 0,
     failed: 0,
     skipped: 0,
+    unknown: 0,
     total: 0,
     duration: 0,
     suites: [],
@@ -21,16 +22,17 @@ export class TestResultsSummarizer implements TestConsumerShape {
     this.summary.suites.push(result);
     this.summary.failed += result.failed;
     this.summary.passed += result.passed;
+    this.summary.unknown += result.unknown;
     this.summary.skipped += result.skipped;
     this.summary.duration += result.duration;
-    this.summary.total += (result.failed + result.passed + result.skipped);
+    this.summary.total += result.total;
   }
 
   /**
    * Merge all test results into a single Suite Result
    */
   onEvent(event: TestEvent): void {
-    if (event.phase === 'after' && event.type === 'suite') {
+    if (event.type === 'suite' && event.phase === 'after') {
       this.#merge(event.suite);
     }
   }

package/src/consumer/types/tap-summary.ts

@@ -99,16 +99,18 @@ export class TapSummaryEmitter implements TestConsumerShape {
         files.set(file, { key: file, duration: 0, tests: 0 });
       }
 
+      const testCount = Object.keys(event.suite.tests).length;
+
       suites.set(event.suite.classId, {
         key: event.suite.classId,
         duration: event.suite.duration,
-        tests: event.suite.tests.length
+        tests: testCount
       });
 
       files.get(file)!.duration += event.suite.duration;
-      files.get(file)!.tests += event.suite.tests.length;
+      files.get(file)!.tests += testCount;
       modules.get(module)!.duration += event.suite.duration;
-      modules.get(module)!.tests += event.suite.tests.length;
+      modules.get(module)!.tests += testCount;
     }
   }
 

package/src/consumer/types.ts

@@ -1,7 +1,7 @@
 import { Class } from '@travetto/runtime';
 
-import { TestEvent } from '../model/event.ts';
-import { Counts, SuiteResult } from '../model/suite.ts';
+import type { TestEvent, TestRemoveEvent } from '../model/event.ts';
+import type { Counts, SuiteResult } from '../model/suite.ts';
 
 /**
  * All suite results
@@ -52,6 +52,10 @@ export interface TestConsumerShape {
    * Handle individual tests events
    */
   onEvent(event: TestEvent): void;
+  /**
+   * Handle when a remove event is fired
+   */
+  onRemoveEvent?(event: TestRemoveEvent): void;
   /**
    * Summarize all results
    */