detox 20.0.1-breaking.new-global-lifecycle.0 → 20.0.1

package/src/logger/utils/DetoxLogFinalizer.js

@@ -0,0 +1,140 @@
+const path = require('path');
+const { promisify } = require('util');
+
+const fs = require('fs-extra');
+const glob = require('glob');
+const pipe = require('multipipe');
+
+const temporary = require('../../artifacts/utils/temporaryPath');
+
+const globAsync = promisify(glob);
+const globSync = glob.sync;
+const streamUtils = () => require('../../logger/utils/streamUtils');
+
+/**
+ * @typedef DetoxLogFinalizerConfig
+ * @property {import('../../ipc/SessionState')} session
+ */
+
+class DetoxLogFinalizer {
+  /** @param {DetoxLogFinalizerConfig} config */
+  constructor(config) {
+    this._session = config.session;
+  }
+
+  createEventStream() {
+    const sessionId = this._session.id;
+    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+
+    return streamUtils()
+      .uniteSessionLogs(logs)
+      .pipe(streamUtils().chromeTraceStream());
+  }
+
+  async finalize() {
+    const sessionId = this._session.id;
+    const logs = await globAsync(temporary.for.jsonl(`${sessionId}.*`));
+    if (logs.length === 0) {
+      return;
+    }
+
+    if (this._areLogsEnabled()) {
+      const rootDir = this._config.artifacts.rootDir;
+
+      await fs.mkdirp(rootDir);
+      const [out1Stream, out2Stream] = ['detox.log', 'detox.trace.json']
+        .map((filename) => fs.createWriteStream(path.join(rootDir, filename)));
+
+      const tidHashMap = await this._scanForThreadIds(logs);
+      const mergedStream = streamUtils().uniteSessionLogs(logs);
+
+      await Promise.all([
+        pipe(mergedStream, streamUtils().debugStream(this._config.logger.options), out1Stream),
+        pipe(mergedStream, streamUtils().chromeTraceStream(tidHashMap), streamUtils().writeJSON(), out2Stream),
+      ]);
+    }
+
+    await Promise.all(logs.map(filepath => fs.remove(filepath)));
+  }
+
+  finalizeSync() {
+    const sessionId = this._session.id;
+    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+    if (logs.length === 0) {
+      return;
+    }
+
+    const logsEnabled = this._areLogsEnabled();
+    const rootDir = logsEnabled ? this._config.artifacts.rootDir : '';
+    if (logsEnabled) {
+      fs.mkdirpSync(rootDir);
+    }
+
+    for (const log of logs) {
+      if (logsEnabled) {
+        fs.moveSync(log, path.join(rootDir, path.basename(log)));
+      } else {
+        fs.removeSync(log);
+      }
+    }
+  }
+
+  /** @private */
+  get _config() {
+    // The config appears later in the lifecycle, so we need to access it lazily
+    return this._session.detoxConfig;
+  }
+
+  /** @private */
+  _areLogsEnabled() {
+    if (!this._config) {
+      return false;
+    }
+
+    const { rootDir, plugins } = this._config.artifacts;
+    if (!rootDir || !plugins) {
+      return false;
+    }
+
+    if (!plugins.log.enabled) {
+      return false;
+    }
+
+    if (!plugins.log.keepOnlyFailedTestsArtifacts) {
+      return true;
+    }
+
+    return this._session.testResults.some(r => !r.success);
+  }
+
+  /** @private */
+  async _scanForThreadIds(logs) {
+    const processes = await new Promise((resolve, reject) => {
+      const result = {};
+      streamUtils().uniteSessionLogs(logs)
+        .on('end', () => resolve(result))
+        .on('error', /* istanbul ignore next */ (err) => reject(err))
+        .on('data', (event) => {
+          const { ph, pid, tid, cat } = event;
+          if (ph === 'B' || ph === 'i') {
+            const categories = (result[pid] = result[pid] || {});
+            const mainCategory = String(cat).split(',')[0];
+            const tids = (categories[mainCategory] = categories[mainCategory] || []);
+            if (!tids.includes(tid)) {
+              tids.push(tid);
+            }
+          }
+        });
+    });
+
+    const tidArray = Object.entries(processes).flatMap(([pid, categories]) => {
+      return Object.entries(categories).flatMap(([category, tids]) => {
+        return tids.map(tid => `${pid}:${category}:${tid}`);
+      });
+    });
+
+    return new Map(tidArray.map((hash, index) => [hash, index]));
+  }
+}
+
+module.exports = DetoxLogFinalizer;

package/src/logger/utils/MessageStack.js

@@ -0,0 +1,24 @@
+class MessageStack {
+  constructor() {
+    this._map = {};
+  }
+
+  push(tid, msg) {
+    if (this._map[tid] == null) {
+      this._map[tid] = [];
+    }
+
+    return this._map[tid].push(msg);
+  }
+
+  pop(tid) {
+    const stack = this._map[tid];
+    if (stack == null || stack.length === 0) {
+      return ['<no begin message>'];
+    }
+
+    return stack.pop();
+  }
+}
+
+module.exports = MessageStack;

package/src/logger/utils/ThreadDispatcher.js

@@ -0,0 +1,61 @@
+const isUndefined = (x) => x === undefined;
+
+class ThreadDispatcher {
+  /**
+   * @param {string} name
+   */
+  constructor(name) {
+    this.name = name;
+    this.stacks = [];
+    this.threads = [];
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  begin(id) {
+    const tid = this._findTID(id);
+    this.threads[tid] = id;
+    this.stacks[tid] = (this.stacks[tid] || 0) + 1;
+    return tid;
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  resolve(id) {
+    return this._findTID(id);
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  end(id) {
+    const tid = this._findTID(id);
+    if (this.stacks[tid] && --this.stacks[tid] === 0) {
+      delete this.threads[tid];
+    }
+    return tid;
+  }
+
+  /**
+   * @param {string | number | undefined} id
+   * @returns {number}
+   * @private
+   *
+   * Memory-efficient finder of a free item index in the threads array.
+   */
+  _findTID(id) {
+    let tid = this.threads.indexOf(id);
+    if (tid === -1) {
+      // Try to find a recently released slot in the array:
+      tid = this.threads.findIndex(isUndefined);
+    }
+    return tid === -1 ? this.threads.length : tid;
+  }
+}
+
+module.exports = ThreadDispatcher;

package/src/logger/{customConsoleLogger.js → utils/customConsoleLogger.js}

@@ -1,6 +1,6 @@
 const util = require('util');
 
-const callsites = require('../utils/callsites');
+const callsites = require('../../utils/callsites');
 
 const USER_STACK_FRAME_INDEX = 2;
 
@@ -25,20 +25,20 @@ function getStackDump() {
 
 function proxyLog(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n', util.format(...args));
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, util.format(...args));
   };
 }
 
 function proxyTracing(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n  Trace:', util.format(...args), '\n\r' + getStackDump());
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin(), stack: getStackDump() }, util.format(...args));
   };
 }
 
 function proxyAssert(bunyanLoggerFn) {
   return (condition, ...args) => {
     if (!condition) {
-      bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n  AssertionError:', util.format(...args));
+      bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, 'AssertionError:', util.format(...args));
     }
   };
 }
@@ -48,6 +48,7 @@ function overrideConsoleMethods(console, bunyanLogger) {
     const log = bunyanLogger;
 
     override(console, 'log', log.info.bind(log));
+    override(console, 'info', log.info.bind(log));
     override(console, 'warn', log.warn.bind(log));
     override(console, 'trace', log.info.bind(log));
     override(console, 'error', log.error.bind(log));

package/src/logger/utils/sanitizeBunyanContext.js

@@ -0,0 +1,30 @@
+const _ = require('lodash');
+
+const RESERVED_PROPERTIES = [
+  'hostname',
+  'level',
+  'msg',
+  'name',
+  'pid',
+  'tid',
+  'ph',
+  'time',
+];
+
+function hasProperty(p) {
+  return _.has(this, p);
+}
+
+function hasReservedProperties(o) {
+  return RESERVED_PROPERTIES.some(hasProperty, o); // eslint-disable-line unicorn/no-array-method-this-argument
+}
+
+function escapeCallback(value, key) {
+  return RESERVED_PROPERTIES.includes(key) ? `${key}$` : key;
+}
+
+function sanitizeBunyanContext(context) {
+  return context && hasReservedProperties(context) ? _.mapKeys(context, escapeCallback) : context;
+}
+
+module.exports = sanitizeBunyanContext;

package/src/{utils → logger/utils}/streamUtils.js

@@ -2,15 +2,14 @@ const { PassThrough, Transform } = require('stream');
 
 const bunyanDebugStream = require('bunyan-debug-stream');
 const duplexify = require('duplexify');
+const fs = require('fs-extra');
 const multiSort = require('multi-sort-stream');
 const pipe = require('multipipe');
 const JsonlParser = require('stream-json/jsonl/Parser');
 const stripAnsi = require('strip-ansi');
 const { AbstractEventBuilder } = require('trace-event-lib');
 
-const DetoxTracer = require('../logger/DetoxTracer');
-
-const log = require('./logger').child({ __filename });
+const log = require('../../utils/logger').child({ cat: 'logger' });
 
 function compareTimestamps(a, b) {
   return +(a.value.time > b.value.time) - +(a.value.time < b.value.time);
@@ -136,37 +135,60 @@ class JsonlStringer extends Transform {
 
 class SimpleEventBuilder extends AbstractEventBuilder {
   events = [];
+
   send(event) {
     this.events.push(event);
   }
 }
 
-function chromeTraceStream() {
+const ERROR_TID = 37707;
+
+function getTidHash({ pid, tid, cat }) {
+  const mainCategory = String(cat).split(',')[0];
+  return `${pid}:${mainCategory}:${tid}`;
+}
+
+function getFixedTID(tidHashMap, event) {
+  let tid = tidHashMap.get(getTidHash(event));
+  if (tid === undefined) {
+    tid = ERROR_TID;
+  }
+
+  return tid;
+}
+
+/**
+ * @param {Map<string, number>} [tidHashMap]
+ */
+function chromeTraceStream(tidHashMap) {
   const knownPids = new Set();
   const knownTids = new Set();
+  const fixTID = tidHashMap ? getFixedTID.bind(null, tidHashMap) : (x) => x.tid;
 
   return flatMapTransform((data) => {
     // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
-    const { pid, trace, msg, time, name: _name, hostname: _hostname, ...args } = data;
-    const tid = trace ? trace.tid : DetoxTracer.threadize(args.cat);
+    const { cat: rawCat, msg: name, ph = 'i', pid, tid: _tid, time, name: _name, hostname: _hostname, ...args } = data;
     const ts = new Date(time).getTime() * 1E3;
+    const cat = rawCat || 'undefined';
+    const tid = fixTID(data);
 
     const builder = new SimpleEventBuilder();
     if (!knownPids.has(pid)) {
-      builder.process_name(pid === process.pid ? 'primary' : 'secondary', pid);
+      builder.metadata({ pid, ts, name: 'process_name', args: { name: pid === process.pid ? 'primary' : 'secondary' } });
+      builder.metadata({ pid, ts, name: 'process_sort_index', args: { sort_index: knownPids.size } });
       knownPids.add(pid);
     }
 
-    const tidHash = `${pid}:${tid}`;
-    if (!knownTids.has(tidHash)) {
-      builder.thread_name(DetoxTracer.categorize(tid), tid, pid);
-      builder.thread_sort_index(tid, tid, pid);
-      knownTids.add(tidHash);
+    if (!knownTids.has(tid)) {
+      const primaryCategory = cat.split(',', 1)[0];
+      builder.metadata({ pid, tid, ts, name: 'thread_name', args: { name: primaryCategory } });
+      builder.metadata({ pid, tid, ts, name: 'thread_sort_index', args: { sort_index: tid } });
+      knownTids.add(tid);
     }
 
-    const event = { ph: 'i', ...data.trace, pid, tid, ts, args };
-    if (!trace || trace.ph !== 'E') {
-      event.name = msg || '';
+    const event = { ph, name, pid, tid, cat, ts, args };
+    if (ph === 'E') {
+      delete event.name;
     }
 
     builder.events.push(event);
@@ -190,11 +212,11 @@ function preventErrorSubscriptions(emitter) {
 }
 
 function readJSONL() {
-  const readable = new PassThrough({ objectMode: true });
+  const readable = through();
   const writable = JsonlParser.make({ checkErrors: true })
     .on('error', (err) => {
       if (err instanceof SyntaxError) {
-        log.debug({ event: 'JSONL_ERROR', err });
+        log.debug({ err });
         readable.end();
       } else {
         readable.emit('error', err);
@@ -204,6 +226,17 @@ function readJSONL() {
   return duplexify.obj(preventErrorSubscriptions(writable), writable.pipe(readable));
 }
 
+/**
+ * @param {string[]} logs
+ * @returns {NodeJS.ReadableStream}
+ */
+function uniteSessionLogs(logs) {
+  const readable = through();
+  const jsonlStreams = logs.map(filePath => fs.createReadStream(filePath).pipe(readJSONL()));
+  mergeSortedJSONL(jsonlStreams).pipe(readable);
+  return readable;
+}
+
 module.exports = {
   readJSONL,
   writeJSON,
@@ -211,4 +244,5 @@ module.exports = {
   mergeSortedJSONL,
   debugStream,
   chromeTraceStream,
+  uniteSessionLogs,
 };

package/src/logger/utils/tracerLegacy.js

@@ -0,0 +1,37 @@
+const methods = {
+  startSection(logger) {
+    return (msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.begin(args, msg);
+      } else {
+        return logger.trace.begin(msg);
+      }
+    };
+  },
+
+  endSection(logger) {
+    return (_msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.end(args);
+      } else {
+        return logger.trace.end();
+      }
+    };
+  },
+
+  traceCall(logger) {
+    return (name, action, args = {}) => logger.trace.complete(args, name, action);
+  },
+};
+
+function installLegacyTracerInterface(logger, target) {
+  target.traceCall = methods.traceCall(logger);
+  target.trace = Object.freeze({
+    startSection: methods.startSection(logger),
+    endSection: methods.endSection(logger),
+  });
+
+  return this;
+}
+
+module.exports = { install: installLegacyTracerInterface };

package/src/realms/DetoxContext.js

@@ -1,48 +1,70 @@
 const funpermaproxy = require('funpermaproxy');
 
+const temporary = require('../artifacts/utils/temporaryPath');
 const { DetoxRuntimeError } = require('../errors');
-const DetoxLogger = require('../logger/DetoxLogger');
-const DetoxTracer = require('../logger/DetoxTracer');
+const { DetoxLogger, DetoxLogFinalizer, installLegacyTracerInterface } = require('../logger');
 const symbols = require('../symbols');
 
 const DetoxConstants = require('./DetoxConstants');
 
+//#region Protected symbols
 const $cleanup = Symbol('cleanup');
-const $logger = Symbol('logger');
+const $logFinalizer = Symbol('logFinalizer');
 const $restoreSessionState = Symbol('restoreSessionState');
 const $sessionState = Symbol('restoreSessionState');
-const $tracer = Symbol('tracer');
-
-const _worker = Symbol('worker');
+const $status = Symbol('status');
+const $worker = Symbol('worker');
+//#endregion
 
 class DetoxContext {
   constructor() {
-    this[symbols.globalSetup] = this[symbols.globalSetup].bind(this);
-    this[symbols.globalTeardown] = this[symbols.globalTeardown].bind(this);
-    this[symbols.reportFailedTests] = this[symbols.reportFailedTests].bind(this);
+    /** @type {DetoxInternals.DetoxStatus} */
+    this[$status] = 'inactive';
+
+    const _init = this[symbols.init].bind(this);
+    this[symbols.init] = async (opts) => {
+      this[$status] = 'init';
+      await _init(opts);
+      this[$status] = 'active';
+    };
+
+    const _cleanup = this[symbols.cleanup].bind(this);
+    this[symbols.cleanup] = async () => {
+      this[$status] = 'cleanup';
+      try {
+        await _cleanup();
+      } finally {
+        this[$status] = 'inactive';
+      }
+    };
+
+    this[symbols.getStatus] = this[symbols.getStatus].bind(this);
+    this[symbols.reportTestResults] = this[symbols.reportTestResults].bind(this);
     this[symbols.resolveConfig] = this[symbols.resolveConfig].bind(this);
-    this[symbols.setup] = this[symbols.setup].bind(this);
-    this[symbols.teardown] = this[symbols.teardown].bind(this);
+    this[symbols.installWorker] = this[symbols.installWorker].bind(this);
+    this[symbols.uninstallWorker] = this[symbols.uninstallWorker].bind(this);
 
     this[$sessionState] = this[$restoreSessionState]();
 
-    const loggerConfig = this[$sessionState].detoxConfig
-      ? this[$sessionState].detoxConfig.loggerConfig
-      : undefined;
-
     /**
-     * @protected
      * @type {DetoxLogger & Detox.Logger}
      */
-    this[$logger] = new DetoxLogger(loggerConfig);
-    /** @protected */
-    this[$tracer] = DetoxTracer.default({
-      logger: this[$logger],
+    this[symbols.logger] = new DetoxLogger({
+      file: temporary.for.jsonl(`${this[$sessionState].id}.${process.pid}`),
+      userConfig: this[$sessionState].detoxConfig
+        ? this[$sessionState].detoxConfig.logger
+        : null,
     });
-    /** @deprecated */
-    this.traceCall = this[$tracer].bind(this[$tracer]);
+
+    this.log = this[symbols.logger].child({ cat: 'user' });
+    installLegacyTracerInterface(this.log, this);
+
+    this[$logFinalizer] = new DetoxLogFinalizer({
+      session: this[$sessionState]
+    });
+
     /** @type {import('../DetoxWorker') | null} */
-    this[_worker] = null;
+    this[$worker] = null;
   }
 
   //#region Public members
@@ -62,91 +84,81 @@ class DetoxContext {
     return DetoxConstants;
   }
 
-  /**
-   * @returns {Detox.Logger}
-   */
-  get log() {
-    return this[$logger];
-  }
-
-  /**
-   * @returns {Detox.Tracer}
-   */
-  get trace() {
-    return this[$tracer];
-  }
   //#endregion
 
   //#region Internal members
-  [symbols.onRunStart] = (...args) => this[symbols.worker].onRunStart(...args);
   [symbols.onRunDescribeStart] = (...args) => this[symbols.worker].onRunDescribeStart(...args);
   [symbols.onTestStart] = (...args) => this[symbols.worker].onTestStart(...args);
-  [symbols.onHookStart] = (...args) => this[symbols.worker].onHookStart(...args);
   [symbols.onHookFailure] = (...args) => this[symbols.worker].onHookFailure(...args);
-  [symbols.onHookSuccess] = (...args) => this[symbols.worker].onHookSuccess(...args);
-  [symbols.onTestFnStart] = (...args) => this[symbols.worker].onTestFnStart(...args);
   [symbols.onTestFnFailure] = (...args) => this[symbols.worker].onTestFnFailure(...args);
-  [symbols.onTestFnSuccess] = (...args) => this[symbols.worker].onTestFnSuccess(...args);
   [symbols.onTestDone] = (...args) => this[symbols.worker].onTestDone(...args);
   [symbols.onRunDescribeFinish] = (...args) => this[symbols.worker].onRunDescribeFinish(...args);
-  [symbols.onRunFinish] = (...args) => this[symbols.worker].onRunFinish(...args);
   [symbols.config] = funpermaproxy(() => this[symbols.session].detoxConfig);
   [symbols.session] = funpermaproxy(() => this[$sessionState]);
+  [symbols.tracing] = Object.freeze({
+    createEventStream: () => this[$logFinalizer].createEventStream(),
+  });
   /** @abstract */
-  [symbols.reportFailedTests](_testFilePaths) {}
+  [symbols.reportTestResults](_testResults) {}
   /**
    * @abstract
-   * @param {Partial<DetoxInternals.DetoxGlobalSetupOptions>} _opts
+   * @param {Partial<DetoxInternals.DetoxInitOptions>} _opts
    * @returns {Promise<DetoxInternals.RuntimeConfig>}
    */
   async [symbols.resolveConfig](_opts) { return null; }
 
+  [symbols.getStatus]() {
+    return this[$status];
+  }
+
   get [symbols.worker]() {
-    if (!this[_worker]) {
+    if (!this[$worker]) {
       throw new DetoxRuntimeError({
-        message: `Detox worker instance has not been initialized in this context (${this.constructor.name}).`,
-        hint: DetoxRuntimeError.reportIssueIfJest + '\n' + 'Otherwise, make sure you call detox.setup() beforehand.',
+        message: `Detox worker instance has not been installed in this context (${this.constructor.name}).`,
+        hint: DetoxRuntimeError.reportIssueIfJest + '\n' + 'Otherwise, make sure you call detox.installWorker() beforehand.',
       });
     }
 
-    return this[_worker];
+    return this[$worker];
   }
 
   /**
    * @abstract
-   * @param {Partial<DetoxInternals.DetoxGlobalSetupOptions>} [_opts]
+   * @param {Partial<DetoxInternals.DetoxInitOptions>} [_opts]
    * @returns {Promise<void>}
    */
-  async [symbols.globalSetup](_opts = {}) {}
+  async [symbols.init](_opts = {}) {}
 
   /**
-   * @param {Partial<DetoxInternals.DetoxConfigurationSetupOptions>} [opts]
+   * @param {Partial<DetoxInternals.DetoxInstallWorkerOptions>} [opts]
    */
-  async [symbols.setup](opts) {
-    const theGlobal = opts.global || global;
-    theGlobal['__detox__'] = this;
-    this[$logger].overrideConsole(theGlobal);
+  async [symbols.installWorker](opts) {
+    if (opts.global) {
+      opts.global['__detox__'] = this;
+      this.log.overrideConsole(opts.global);
+    }
 
     const DetoxWorker = require('../DetoxWorker');
-    DetoxWorker.global = theGlobal;
-    this[_worker] = new DetoxWorker(this);
-    await this[_worker].init();
+    DetoxWorker.global = opts.global || global;
+    this[$worker] = new DetoxWorker(this);
+    this[$worker].id = opts.workerId;
+    await this[$worker].init();
   }
 
-  async [symbols.teardown]() {
+  async [symbols.uninstallWorker]() {
     try {
-      if (this[_worker]) {
-        await this[_worker].cleanup();
+      if (this[$worker]) {
+        await this[$worker].cleanup();
       }
     } finally {
-      this[_worker] = null;
+      this[$worker] = null;
     }
   }
 
   /**
    * @abstract
    */
-  async [symbols.globalTeardown]() {}
+  async [symbols.cleanup]() {}
   //#endregion
 
   //#region Protected members
@@ -163,8 +175,9 @@ class DetoxContext {
 module.exports = DetoxContext;
 module.exports.protected = {
   $cleanup,
-  $logger,
+  $logFinalizer,
   $restoreSessionState,
+  $status,
   $sessionState,
-  $tracer,
+  $worker,
 };