detox 20.0.2-breaking.new-global-lifecycle.0 → 20.0.2

package/src/logger/utils/BunyanLogger.js

@@ -0,0 +1,115 @@
+const { PassThrough } = require('stream');
+
+const bunyan = require('bunyan');
+const bds = require('bunyan-debug-stream');
+const _ = require('lodash');
+
+const { DetoxInternalError } = require('../../errors');
+
+class BunyanLogger {
+  constructor() {
+    this._bunyan = bunyan.createLogger({ name: 'detox', streams: [] });
+    /** @type {bunyan.Stream} */
+    this._debugStream = null;
+    /** @type {bunyan.Stream} */
+    this._fileStream = null;
+  }
+
+  /**
+   * @returns {bunyan}
+   */
+  get logger() {
+    return this._bunyan;
+  }
+
+  /**
+   * @param {Detox.DetoxLoggerConfig} config
+   * @returns {this}
+   */
+  installDebugStream(config) {
+    if (this._debugStream) {
+      _.remove(this._bunyan['streams'], this._debugStream);
+      // @ts-ignore
+      this._debugStream.stream.end();
+      this._debugStream = null;
+    }
+
+    const streamOptions = { out: null, ...config.options };
+    /* istanbul ignore next */
+    if (!streamOptions.out) {
+      // This is a default if-else branch, used everywhere except for the unit tests.
+      streamOptions.out = new PassThrough().pipe(process.stderr);
+    }
+
+    this._debugStream = {
+      type: 'raw',
+      level: config.level,
+      stream: bds.default(streamOptions),
+    };
+
+    this._bunyan.addStream(this._debugStream);
+    return this;
+  }
+
+  /**
+   * @param {string} file
+   * @returns {this}
+   */
+  installFileStream(file) {
+    /* istanbul ignore next */
+    if (this._fileStream) {
+      // This is an impossible condition, but we keep it here for the sake of completeness.
+      throw new DetoxInternalError('Trying to install a second file stream inside already initialized Bunyan logger');
+    }
+
+    this._fileStream = {
+      level: 'trace',
+      path: file,
+    };
+
+    this._bunyan.addStream(this._fileStream);
+
+    return this;
+  }
+
+  async closeFileStreams() {
+    const { _closeStream } = BunyanLogger;
+    const internalBunyanStreams = this._bunyan['streams'];
+    const openFileStreams = _.filter(internalBunyanStreams, this._isOpenFileStream);
+    _.remove(internalBunyanStreams, openFileStreams);
+
+    await Promise.all(openFileStreams.map(_closeStream));
+  }
+
+  /** @private */
+  _isOpenFileStream = (bunyanStream) => {
+    switch (bunyanStream.type) {
+      case 'file':
+        return bunyanStream.path && !bunyanStream.stream.destroyed;
+      case 'raw':
+        /* istanbul ignore next */
+        const stream = bunyanStream.stream === this._debugStream.stream
+          ? bunyanStream.stream._out
+          : bunyanStream.stream;
+
+        /* istanbul ignore next */
+        return stream.fd !== undefined && !stream.closed;
+    }
+  };
+
+  /** @private */
+  static _closeStream(bunyanStream) {
+    return new Promise((resolve, reject) => {
+      bunyanStream.stream.end((err) => {
+        /* istanbul ignore next */
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+}
+
+module.exports = BunyanLogger;

package/src/logger/utils/CategoryThreadDispatcher.js

@@ -0,0 +1,37 @@
+const ThreadDispatcher = require('./ThreadDispatcher');
+const getMainCategory = require('./getMainCategory');
+
+class CategoryThreadDispatcher {
+  constructor() {
+    /** @type {Record<string, ThreadDispatcher>} */
+    this._dispatchers = {};
+  }
+
+  /**
+   * @param {'B' | 'E' | 'i'} ph
+   * @param {string[] | undefined} cat
+   * @param {string | number} id
+   * @returns {number}
+   */
+  resolve(ph, cat, id) {
+    const dispatcher = this._resolveDispatcher(cat);
+
+    switch (ph) {
+      case 'B': return dispatcher.begin(id);
+      case 'E': return dispatcher.end(id);
+      default: return dispatcher.resolve(id);
+    }
+  }
+
+  /** @returns {ThreadDispatcher} */
+  _resolveDispatcher(cat) {
+    const mainCategory = getMainCategory(cat);
+    if (!this._dispatchers[mainCategory]) {
+      this._dispatchers[mainCategory] = new ThreadDispatcher(mainCategory);
+    }
+
+    return this._dispatchers[mainCategory];
+  }
+}
+
+module.exports = CategoryThreadDispatcher;

package/src/logger/utils/DetoxLogFinalizer.js

@@ -0,0 +1,162 @@
+const path = require('path');
+
+const fs = require('fs-extra');
+
+const temporary = require('../../artifacts/utils/temporaryPath');
+
+const { BunyanTransformer, ChromeTraceTransformer } = require('./streams');
+
+/**
+ * @typedef DetoxLogFinalizerConfig
+ * @property {import('../../ipc/SessionState')} session
+ * @property {import('../DetoxLogger')} logger
+ */
+
+class DetoxLogFinalizer {
+  /** @param {DetoxLogFinalizerConfig} config */
+  constructor(config) {
+    this._session = config.session;
+    this._bunyanTransformer = new BunyanTransformer(
+      config.logger.child({ cat: 'logger' }),
+    );
+    this._chromeTransformer = new ChromeTraceTransformer();
+  }
+
+  createEventStream() {
+    const sessionId = this._session.id;
+    const logs = temporary.find.jsonl.sync(`${sessionId}.*`);
+
+    const toChromeTrace = this._chromeTransformer.createStream();
+    return this._bunyanTransformer.uniteSessionLogs(logs).pipe(toChromeTrace);
+  }
+
+  async finalize() {
+    const sessionId = this._session.id;
+    const logs = await temporary.find.jsonl.async(`${sessionId}.*`);
+    if (logs.length === 0) {
+      return;
+    }
+
+    if (this._areLogsEnabled()) {
+      const rootDir = this._config.artifacts.rootDir;
+
+      await fs.mkdirp(rootDir);
+
+      const firstPass = this._bunyanTransformer.uniteSessionLogs(logs);
+      await this._chromeTransformer.scanThreadIDs(firstPass);
+
+      const secondPass = this._bunyanTransformer.uniteSessionLogs(logs);
+      const outStreams = [this._createPlainFileStream(), this._createChromeTraceStream()];
+
+      await Promise.all(outStreams.map(stream => {
+        return new Promise((resolve, reject) => {
+          stream.target
+            .on('finish', resolve)
+            .on('error', reject);
+
+          secondPass.pipe(stream.writable);
+        });
+      }));
+    }
+
+    await Promise.all(logs.map(filepath => fs.remove(filepath)));
+  }
+
+  finalizeSync() {
+    const sessionId = this._session.id;
+    const logs = temporary.find.jsonl.sync(`${sessionId}.*`);
+    if (logs.length === 0) {
+      return;
+    }
+
+    const logsEnabled = this._areLogsEnabled();
+    const rootDir = logsEnabled ? this._config.artifacts.rootDir : '';
+    if (logsEnabled) {
+      fs.mkdirpSync(rootDir);
+    }
+
+    for (const log of logs) {
+      if (logsEnabled) {
+        const dest = path.join(rootDir, path.basename(log));
+        this._safeMoveSync(log, dest);
+      } else {
+        this._safeRemoveSync(log);
+      }
+    }
+  }
+
+  _createPlainFileStream() {
+    const rootDir = this._config.artifacts.rootDir;
+    const bunyanOptions = this._config.logger.options;
+    const transformer = this._bunyanTransformer.createPlainTransformer(bunyanOptions);
+    const fileStream = fs.createWriteStream(path.join(rootDir, 'detox.log'));
+    transformer.readable.pipe(fileStream);
+
+    return { writable: transformer.writable, target: fileStream };
+  }
+
+  _createChromeTraceStream() {
+    const rootDir = this._config.artifacts.rootDir;
+    const transformer = this._chromeTransformer.createSerializedStream();
+    const fileStream = fs.createWriteStream(path.join(rootDir, 'detox.trace.json'));
+    transformer.readable.pipe(fileStream);
+
+    return { writable: transformer.writable, target: fileStream };
+  }
+
+  /* istanbul ignore next */
+  _safeMoveSync(src, dest) {
+    // Using console.* instead of logger.* because this is the end of the execution,
+    // and the logger is already closed.
+
+    try {
+      fs.moveSync(src, dest);
+    } catch (moveError) {
+      console.warn(`Failed to move a log file from: ${src}.\nReason: ${moveError.message}`);
+      try {
+        fs.copySync(src, dest);
+      } catch (copyError) {
+        console.warn(`Attempt to copy the file also failed.\nReason: ${copyError.message}`);
+      }
+    }
+  }
+
+  /* istanbul ignore next */
+  _safeRemoveSync(filepath) {
+    try {
+      fs.removeSync(filepath);
+    } catch (removeError) {
+      console.warn(`Failed to remove a log file at: ${filepath}.\nReason: ${removeError.message}`);
+    }
+  }
+
+  /** @private */
+  get _config() {
+    // The config appears later in the lifecycle, so we need to access it lazily
+    return this._session.detoxConfig;
+  }
+
+  /** @private */
+  _areLogsEnabled() {
+    if (!this._config) {
+      return false;
+    }
+
+    const { rootDir, plugins } = this._config.artifacts;
+    if (!rootDir || !plugins) {
+      return false;
+    }
+
+    if (!plugins.log.enabled) {
+      return false;
+    }
+
+    if (!plugins.log.keepOnlyFailedTestsArtifacts) {
+      return true;
+    }
+
+    return this._session.testResults.some(r => !r.success);
+  }
+}
+
+module.exports = DetoxLogFinalizer;

package/src/logger/utils/MessageStack.js

@@ -0,0 +1,35 @@
+const getMainCategory = require('./getMainCategory');
+
+class MessageStack {
+  constructor() {
+    this._map = {};
+  }
+
+  push(context, msg) {
+    const hash = this._hash(context);
+
+    if (this._map[hash] == null) {
+      this._map[hash] = [];
+    }
+
+    return this._map[hash].push(msg);
+  }
+
+  pop(context) {
+    const hash = this._hash(context);
+    const stack = this._map[hash];
+    if (stack == null || stack.length === 0) {
+      return ['<no begin message>'];
+    }
+
+    return stack.pop();
+  }
+
+  _hash(context) {
+    const cat = getMainCategory(context.cat);
+    const tid = context.tid;
+    return `${cat}:${tid}`;
+  }
+}
+
+module.exports = MessageStack;

package/src/logger/utils/ThreadDispatcher.js

@@ -0,0 +1,61 @@
+const isUndefined = (x) => x === undefined;
+
+class ThreadDispatcher {
+  /**
+   * @param {string} name
+   */
+  constructor(name) {
+    this.name = name;
+    this.stacks = [];
+    this.threads = [];
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  begin(id) {
+    const tid = this._findTID(id);
+    this.threads[tid] = id;
+    this.stacks[tid] = (this.stacks[tid] || 0) + 1;
+    return tid;
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  resolve(id) {
+    return this._findTID(id);
+  }
+
+  /**
+   * @param {string | number} [id]
+   * @returns {number}
+   */
+  end(id) {
+    const tid = this._findTID(id);
+    if (this.stacks[tid] && --this.stacks[tid] === 0) {
+      delete this.threads[tid];
+    }
+    return tid;
+  }
+
+  /**
+   * @param {string | number | undefined} id
+   * @returns {number}
+   * @private
+   *
+   * Memory-efficient finder of a free item index in the threads array.
+   */
+  _findTID(id) {
+    let tid = this.threads.indexOf(id);
+    if (tid === -1) {
+      // Try to find a recently released slot in the array:
+      tid = this.threads.findIndex(isUndefined);
+    }
+    return tid === -1 ? this.threads.length : tid;
+  }
+}
+
+module.exports = ThreadDispatcher;

package/src/logger/{customConsoleLogger.js → utils/customConsoleLogger.js}

@@ -1,6 +1,6 @@
 const util = require('util');
 
-const callsites = require('../utils/callsites');
+const callsites = require('../../utils/callsites');
 
 const USER_STACK_FRAME_INDEX = 2;
 
@@ -25,20 +25,20 @@ function getStackDump() {
 
 function proxyLog(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n', util.format(...args));
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, util.format(...args));
   };
 }
 
 function proxyTracing(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n  Trace:', util.format(...args), '\n\r' + getStackDump());
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin(), stack: getStackDump() }, util.format(...args));
   };
 }
 
 function proxyAssert(bunyanLoggerFn) {
   return (condition, ...args) => {
     if (!condition) {
-      bunyanLoggerFn({ event: 'USER_LOG' }, getOrigin(), '\n  AssertionError:', util.format(...args));
+      bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, 'AssertionError:', util.format(...args));
     }
   };
 }
@@ -47,19 +47,36 @@ function overrideConsoleMethods(console, bunyanLogger) {
   if (!console.__detox_log__) {
     const log = bunyanLogger;
 
+    console.__detox_log__ = {
+      log: console.log,
+      info: console.info,
+      warn: console.warn,
+      trace: console.trace,
+      error: console.error,
+      debug: console.debug,
+      assert: console.assert,
+    };
+
     override(console, 'log', log.info.bind(log));
+    override(console, 'info', log.info.bind(log));
     override(console, 'warn', log.warn.bind(log));
     override(console, 'trace', log.info.bind(log));
     override(console, 'error', log.error.bind(log));
     override(console, 'debug', log.debug.bind(log));
     override(console, 'assert', log.error.bind(log));
-
-    console.__detox_log__ = log;
   }
 
   return console;
 }
 
+function restoreConsoleMethods(console) {
+  if (console.__detox_log__) {
+    Object.assign(console, console.__detox_log__);
+    delete console.__detox_log__;
+  }
+}
+
 module.exports = {
   overrideConsoleMethods,
+  restoreConsoleMethods,
 };

package/src/logger/utils/getMainCategory.js

@@ -0,0 +1,5 @@
+function getMainCategory(category) {
+  return category ? String(category).split(',', 1)[0] : 'undefined';
+}
+
+module.exports = getMainCategory;

package/src/logger/utils/sanitizeBunyanContext.js

@@ -0,0 +1,30 @@
+const _ = require('lodash');
+
+const RESERVED_PROPERTIES = [
+  'hostname',
+  'level',
+  'msg',
+  'name',
+  'pid',
+  'tid',
+  'ph',
+  'time',
+];
+
+function hasProperty(p) {
+  return _.has(this, p);
+}
+
+function hasReservedProperties(o) {
+  return RESERVED_PROPERTIES.some(hasProperty, o); // eslint-disable-line unicorn/no-array-method-this-argument
+}
+
+function escapeCallback(value, key) {
+  return RESERVED_PROPERTIES.includes(key) ? `${key}$` : key;
+}
+
+function sanitizeBunyanContext(context) {
+  return context && hasReservedProperties(context) ? _.mapKeys(context, escapeCallback) : context;
+}
+
+module.exports = sanitizeBunyanContext;

package/src/logger/utils/streams/BunyanTransformer.js

@@ -0,0 +1,72 @@
+const fs = require('fs');
+const { PassThrough } = require('stream');
+
+const bunyanDebugStream = require('bunyan-debug-stream');
+const multiSort = require('multi-sort-stream');
+const pipe = require('multipipe');
+const stripAnsi = require('strip-ansi');
+
+const DetoxJSONLParser = require('./DetoxJSONLParser');
+const { mapTransform, through } = require('./transformers');
+
+class BunyanTransformer {
+  /**
+   * @param {Detox.Logger} log
+   */
+  constructor(log) {
+    this._jsonlParser = new DetoxJSONLParser(log);
+  }
+
+  /**
+   * @param {string[]} logFilePaths
+   */
+  uniteSessionLogs(logFilePaths) {
+    const intermediate = mapTransform(BunyanTransformer.normalizeBunyanRecord);
+    const reemitError = (err) => intermediate.emit('error', err);
+
+    const jsonlStreams = logFilePaths.map(filePath => {
+      const { readable, writable } = this._jsonlParser.createTransformer();
+      fs.createReadStream(filePath)
+        .on('error', reemitError)
+        .pipe(writable);
+
+      return readable;
+    });
+
+    const multisorted = multiSort(jsonlStreams, BunyanTransformer.compareTimestamps);
+    return pipe(multisorted, intermediate, through());
+  }
+
+  createPlainTransformer(bunyanOptions) {
+    /** @type {*} */
+    const readable = new PassThrough({ encoding: 'utf8', objectMode: false });
+    const writable = bunyanDebugStream.default({
+      ...bunyanOptions,
+
+      colors: false,
+      out: readable,
+    });
+
+    writable.on('error', /* istanbul ignore next */ (err) => readable.emit('error', err));
+    writable.on('finish', () => readable.end());
+
+    return {
+      writable,
+      readable,
+    };
+  }
+
+  static normalizeBunyanRecord(record) {
+    const value = record.value;
+    value.msg = stripAnsi(value.msg);
+    value.time = new Date(value.time);
+
+    return value;
+  }
+
+  static compareTimestamps(a, b) {
+    return +(a.value.time > b.value.time) - +(a.value.time < b.value.time);
+  }
+}
+
+module.exports = BunyanTransformer;