detox 20.0.14-next-is-hittable-check-2.0 → 20.0.15-prerelease.0

package/src/logger/DetoxLogger.js

@@ -2,7 +2,6 @@ const path = require('path');
 
 const _ = require('lodash');
 
-const temporaryPath = require('../artifacts/utils/temporaryPath');
 const { DetoxInternalError, DetoxError } = require('../errors');
 const { shortFormat } = require('../utils/dateUtils');
 const isPromise = require('../utils/isPromise');
@@ -16,7 +15,7 @@ const sanitizeBunyanContext = require('./utils/sanitizeBunyanContext');
 /**
  * @typedef SharedLoggerConfig
  * @property {string} file
- * @property {Detox.DetoxLoggerConfig} userConfig
+ * @property {Detox.DetoxLoggerConfig} [userConfig]
  * @property {CategoryThreadDispatcher} [dispatcher]
  * @property {BunyanLogger} [bunyan]
  * @property {MessageStack} [messageStack]
@@ -71,25 +70,7 @@ class DetoxLogger {
         .installFileStream(this.file)
         .installDebugStream(this.config);
 
-      this._sharedConfig.dispatcher = new CategoryThreadDispatcher({
-        logger: this,
-        categories: {
-          'lifecycle': [0],
-          'cli': [1],
-          'logger': [2],
-          'ipc': [29],
-          'ws-server': [50, 99],
-          'ws-client': [100, 149],
-          'device': [150, 159],
-          'artifacts-manager': [300],
-          'artifacts-plugin': [310, 349],
-          'artifact': [350, 399],
-          'child-process': [400, 499],
-          'default': [999],
-          'user': [10000, 10999]
-        },
-      });
-
+      this._sharedConfig.dispatcher = new CategoryThreadDispatcher();
       this._sharedConfig.messageStack = new MessageStack();
 
       this.overrideConsole();
@@ -110,7 +91,7 @@ class DetoxLogger {
    * @returns {DetoxLogger}
    */
   child(overrides) {
-    const merged = this._mergeContexts(this._context, overrides);
+    const merged = this._mergeContexts(this._context, sanitizeBunyanContext(overrides));
     return new DetoxLogger(this._sharedConfig, merged);
   }
 
@@ -134,7 +115,6 @@ class DetoxLogger {
     }
 
     _.merge(this.config, config);
-    this._sharedConfig.file = temporaryPath.for.jsonl();
     this._sharedConfig.bunyan.installDebugStream(this.config);
     this.overrideConsole();
   }
@@ -156,14 +136,23 @@ class DetoxLogger {
    */
   _mergeContexts(...contexts) {
     const context = Object.assign({}, ...contexts);
+    const categories = _(contexts).flatMap((c) => {
+      if (c && c.cat) {
+        return _.isArray(c.cat) ? c.cat : c.cat.split(',');
+      }
+
+      return [];
+    }).uniq().join(',');
 
     if (context.error || context.err) {
       context.error = DetoxError.format(context.error || context.err);
       delete context.err;
     }
 
-    if (Array.isArray(context.cat)) {
-      context.cat = context.cat.join(',');
+    if (categories) {
+      context.cat = categories;
+    } else {
+      delete context.cat;
     }
 
     if (context.__filename) {
@@ -171,15 +160,13 @@ class DetoxLogger {
     }
 
     context.ph = context.ph || 'i';
-    context.cat = context.cat || '';
-
     context.tid = this._sharedConfig.dispatcher.resolve(
       context.ph,
       context.cat,
       context.id || 0
     );
 
-    return sanitizeBunyanContext(context);
+    return context;
   }
 
   /**
@@ -200,6 +187,11 @@ class DetoxLogger {
   /** @private */
   _begin(level, ...args) {
     const { context, msg } = this._parseArgs({ ph: 'B' }, args);
+    this._beginInternal(level, context, msg);
+  }
+
+  /** @private */
+  _beginInternal(level, context, msg) {
     this._sharedConfig.messageStack.push(context.tid, msg);
     this._sharedConfig.bunyan.logger[level](context, ...msg);
   }
@@ -207,8 +199,14 @@ class DetoxLogger {
   /** @private */
   _end(level, ...args) {
     let { context, msg } = this._parseArgs({ ph: 'E' }, args);
+    this._endInternal(level, context, msg);
+  }
+
+  /** @private */
+  _endInternal(level, context, msg) {
+    const beginMsg = this._sharedConfig.messageStack.pop(context.tid);
     if (msg.length === 0) {
-      msg = this._sharedConfig.messageStack.pop(context.tid);
+      msg = beginMsg;
     }
 
     this._sharedConfig.bunyan.logger[level](context, ...msg);
@@ -239,14 +237,14 @@ class DetoxLogger {
     });
 
     let result;
-    this[level].begin(context, ...msg);
+    this._beginInternal(level, { ...context, ph: 'B' }, msg);
     try {
       result = typeof action === 'function'
         ? action()
         : action;
 
       if (!isPromise(result)) {
-        end(context);
+        end({ success: true });
       } else {
         result.then(
           () => end({ success: true }),
@@ -268,20 +266,13 @@ class DetoxLogger {
       : _.isObject(args[0])
         ? args[0]
         : undefined;
-    const msg = userContext !== undefined ? args.slice(1) : args;
 
-    if (userContext) {
-      delete userContext.pid;
-      delete userContext.tid;
-      delete userContext.ts;
-      delete userContext.time;
-      delete userContext.ph;
-    }
+    const msg = userContext !== undefined ? args.slice(1) : args;
 
     const context = this._mergeContexts(
       this._context,
       boundContext,
-      userContext,
+      sanitizeBunyanContext(userContext),
     );
 
     return { context, msg };
@@ -298,7 +289,7 @@ class DetoxLogger {
       : undefined;
 
     const cat = level === 'trace' || level === 'debug'
-      ? (value) => require('chalk').yellow((value || '').split(',', 1)[0])
+      ? (value) => require('chalk').yellow(`${value}`.split(',', 1)[0])
       : undefined;
 
     const event = level === 'trace' || level === 'debug'

package/src/logger/index.js

@@ -1,4 +1,5 @@
 module.exports = {
   DetoxLogger: require('./DetoxLogger'),
+  DetoxLogFinalizer: require('./utils/DetoxLogFinalizer'),
   installLegacyTracerInterface: require('./utils/tracerLegacy').install,
 };

package/src/logger/utils/BunyanLogger.js

@@ -27,8 +27,6 @@ class BunyanLogger {
    * @returns {this}
    */
   installDebugStream(config) {
-    const level = config.level || 'info';
-
     if (this._debugStream) {
       _.remove(this._bunyan['streams'], this._debugStream);
       // @ts-ignore
@@ -36,13 +34,17 @@ class BunyanLogger {
       this._debugStream = null;
     }
 
+    const streamOptions = { out: null, ...config.options };
+    /* istanbul ignore next */
+    if (!streamOptions.out) {
+      // This is a default if-else branch, used everywhere except for the unit tests.
+      streamOptions.out = new PassThrough().pipe(process.stderr);
+    }
+
     this._debugStream = {
       type: 'raw',
-      level,
-      stream: bds.default({
-        ...config.options,
-        out: new PassThrough().pipe(process.stderr),
-      }),
+      level: config.level,
+      stream: bds.default(streamOptions),
     };
 
     this._bunyan.addStream(this._debugStream);
@@ -54,7 +56,9 @@ class BunyanLogger {
    * @returns {this}
    */
   installFileStream(file) {
+    /* istanbul ignore next */
     if (this._fileStream) {
+      // This is an impossible condition, but we keep it here for the sake of completeness.
       throw new DetoxInternalError('Trying to install a second file stream inside already initialized Bunyan logger');
     }
 

package/src/logger/utils/CategoryThreadDispatcher.js

@@ -1,23 +1,9 @@
-const _ = require('lodash');
-
 const ThreadDispatcher = require('./ThreadDispatcher');
 
 class CategoryThreadDispatcher {
-  /**
-   * @param {object} config
-   * @param {Record<string, [number, number?]>} config.categories
-   * @param {Detox.Logger} config.logger
-   */
-  constructor(config) {
-    this.categories = config.categories;
-    this.dispatchers = _.mapValues(this.categories, (range, name) => {
-      return new ThreadDispatcher({
-        name,
-        logger: config.logger,
-        min: range[0],
-        max: range[1] || range[0],
-      });
-    });
+  constructor() {
+    /** @type {Record<string, ThreadDispatcher>} */
+    this._dispatchers = {};
   }
 
   /**
@@ -38,20 +24,12 @@ class CategoryThreadDispatcher {
 
   /** @returns {ThreadDispatcher} */
   _resolveDispatcher(cat) {
-    const mainCategory = cat ? cat.split(',', 1)[0] : '';
-    return this.dispatchers[mainCategory] || this.dispatchers.default;
-  }
-
-  categorize(tid) {
-    return _.findKey(this.categories, ([min, max]) => min <= tid && tid <= max) || 'default';
-  }
-
-  threadize(cat) {
-    if (!cat) {
-      return this.categories.default[0];
+    const mainCategory = cat ? cat.split(',', 1)[0] : 'undefined';
+    if (!this._dispatchers[mainCategory]) {
+      this._dispatchers[mainCategory] = new ThreadDispatcher(mainCategory);
     }
 
-    return _.find(this.categories, (_, key) => key === cat[0]);
+    return this._dispatchers[mainCategory];
   }
 }
 

package/src/logger/utils/DetoxLogFinalizer.js

@@ -0,0 +1,140 @@
+const path = require('path');
+const { promisify } = require('util');
+
+const fs = require('fs-extra');
+const glob = require('glob');
+const pipe = require('multipipe');
+
+const temporary = require('../../artifacts/utils/temporaryPath');
+
+const globAsync = promisify(glob);
+const globSync = glob.sync;
+const streamUtils = () => require('../../logger/utils/streamUtils');
+
+/**
+ * @typedef DetoxLogFinalizerConfig
+ * @property {import('../../ipc/SessionState')} session
+ */
+
+class DetoxLogFinalizer {
+  /** @param {DetoxLogFinalizerConfig} config */
+  constructor(config) {
+    this._session = config.session;
+  }
+
+  createEventStream() {
+    const sessionId = this._session.id;
+    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+
+    return streamUtils()
+      .uniteSessionLogs(logs)
+      .pipe(streamUtils().chromeTraceStream());
+  }
+
+  async finalize() {
+    const sessionId = this._session.id;
+    const logs = await globAsync(temporary.for.jsonl(`${sessionId}.*`));
+    if (logs.length === 0) {
+      return;
+    }
+
+    if (this._areLogsEnabled()) {
+      const rootDir = this._config.artifacts.rootDir;
+
+      await fs.mkdirp(rootDir);
+      const [out1Stream, out2Stream] = ['detox.log', 'detox.trace.json']
+        .map((filename) => fs.createWriteStream(path.join(rootDir, filename)));
+
+      const tidHashMap = await this._scanForThreadIds(logs);
+      const mergedStream = streamUtils().uniteSessionLogs(logs);
+
+      await Promise.all([
+        pipe(mergedStream, streamUtils().debugStream(this._config.logger.options), out1Stream),
+        pipe(mergedStream, streamUtils().chromeTraceStream(tidHashMap), streamUtils().writeJSON(), out2Stream),
+      ]);
+    }
+
+    await Promise.all(logs.map(filepath => fs.remove(filepath)));
+  }
+
+  finalizeSync() {
+    const sessionId = this._session.id;
+    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+    if (logs.length === 0) {
+      return;
+    }
+
+    const logsEnabled = this._areLogsEnabled();
+    const rootDir = logsEnabled ? this._config.artifacts.rootDir : '';
+    if (logsEnabled) {
+      fs.mkdirpSync(rootDir);
+    }
+
+    for (const log of logs) {
+      if (logsEnabled) {
+        fs.moveSync(log, path.join(rootDir, path.basename(log)));
+      } else {
+        fs.removeSync(log);
+      }
+    }
+  }
+
+  /** @private */
+  get _config() {
+    // The config appears later in the lifecycle, so we need to access it lazily
+    return this._session.detoxConfig;
+  }
+
+  /** @private */
+  _areLogsEnabled() {
+    if (!this._config) {
+      return false;
+    }
+
+    const { rootDir, plugins } = this._config.artifacts;
+    if (!rootDir || !plugins) {
+      return false;
+    }
+
+    if (!plugins.log.enabled) {
+      return false;
+    }
+
+    if (!plugins.log.keepOnlyFailedTestsArtifacts) {
+      return true;
+    }
+
+    return this._session.testResults.some(r => !r.success);
+  }
+
+  /** @private */
+  async _scanForThreadIds(logs) {
+    const processes = await new Promise((resolve, reject) => {
+      const result = {};
+      streamUtils().uniteSessionLogs(logs)
+        .on('end', () => resolve(result))
+        .on('error', /* istanbul ignore next */ (err) => reject(err))
+        .on('data', (event) => {
+          const { ph, pid, tid, cat } = event;
+          if (ph === 'B' || ph === 'i') {
+            const categories = (result[pid] = result[pid] || {});
+            const mainCategory = String(cat).split(',')[0];
+            const tids = (categories[mainCategory] = categories[mainCategory] || []);
+            if (!tids.includes(tid)) {
+              tids.push(tid);
+            }
+          }
+        });
+    });
+
+    const tidArray = Object.entries(processes).flatMap(([pid, categories]) => {
+      return Object.entries(categories).flatMap(([category, tids]) => {
+        return tids.map(tid => `${pid}:${category}:${tid}`);
+      });
+    });
+
+    return new Map(tidArray.map((hash, index) => [hash, index]));
+  }
+}
+
+module.exports = DetoxLogFinalizer;

package/src/logger/utils/ThreadDispatcher.js

@@ -2,17 +2,10 @@ const isUndefined = (x) => x === undefined;
 
 class ThreadDispatcher {
   /**
-   * @param {object} options
-   * @param {Detox.Logger} options.logger
-   * @param {string} options.name
-   * @param {number} options.min
-   * @param {number} [options.max]
+   * @param {string} name
    */
-  constructor({ logger, name, min, max = Infinity }) {
-    this.logger = logger;
+  constructor(name) {
     this.name = name;
-    this.min = min;
-    this.max = max;
     this.stacks = [];
     this.threads = [];
   }
@@ -25,7 +18,7 @@ class ThreadDispatcher {
     const tid = this._findTID(id);
     this.threads[tid] = id;
     this.stacks[tid] = (this.stacks[tid] || 0) + 1;
-    return this._transpose(tid);
+    return tid;
   }
 
   /**
@@ -33,8 +26,7 @@ class ThreadDispatcher {
    * @returns {number}
    */
   resolve(id) {
-    const tid = this._findTID(id);
-    return this._transpose(tid);
+    return this._findTID(id);
   }
 
   /**
@@ -46,7 +38,7 @@ class ThreadDispatcher {
     if (this.stacks[tid] && --this.stacks[tid] === 0) {
       delete this.threads[tid];
     }
-    return this._transpose(tid);
+    return tid;
   }
 
   /**
@@ -64,18 +56,6 @@ class ThreadDispatcher {
     }
     return tid === -1 ? this.threads.length : tid;
   }
-
-  _transpose(id) {
-    const result = this.min + id;
-    if (result > this.max) {
-      this.logger.warn(
-        { cat: ['logger', 'thread-dispatcher'] },
-        `${this.name} trace thread dispatcher has run out of available thread IDs: ${this.min}..${this.max}`
-      );
-    }
-
-    return Math.min(result, this.max);
-  }
 }
 
 module.exports = ThreadDispatcher;

package/src/logger/utils/customConsoleLogger.js

@@ -25,20 +25,20 @@ function getStackDump() {
 
 function proxyLog(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ origin: getOrigin() }, util.format(...args));
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, util.format(...args));
   };
 }
 
 function proxyTracing(bunyanLoggerFn) {
   return (...args) => {
-    bunyanLoggerFn({ origin: getOrigin(), stack: getStackDump() }, util.format(...args));
+    bunyanLoggerFn({ cat: 'user', origin: getOrigin(), stack: getStackDump() }, util.format(...args));
   };
 }
 
 function proxyAssert(bunyanLoggerFn) {
   return (condition, ...args) => {
     if (!condition) {
-      bunyanLoggerFn({ origin: getOrigin() }, 'AssertionError:', util.format(...args));
+      bunyanLoggerFn({ cat: 'user', origin: getOrigin() }, 'AssertionError:', util.format(...args));
     }
   };
 }
@@ -48,6 +48,7 @@ function overrideConsoleMethods(console, bunyanLogger) {
     const log = bunyanLogger;
 
     override(console, 'log', log.info.bind(log));
+    override(console, 'info', log.info.bind(log));
     override(console, 'warn', log.warn.bind(log));
     override(console, 'trace', log.info.bind(log));
     override(console, 'error', log.error.bind(log));

package/src/logger/utils/sanitizeBunyanContext.js

@@ -6,6 +6,8 @@ const RESERVED_PROPERTIES = [
   'msg',
   'name',
   'pid',
+  'tid',
+  'ph',
   'time',
 ];
 
@@ -22,7 +24,7 @@ function escapeCallback(value, key) {
 }
 
 function sanitizeBunyanContext(context) {
-  return hasReservedProperties(context) ? _.mapKeys(context, escapeCallback) : context;
+  return context && hasReservedProperties(context) ? _.mapKeys(context, escapeCallback) : context;
 }
 
 module.exports = sanitizeBunyanContext;

package/src/logger/utils/streamUtils.js

@@ -3,14 +3,12 @@ const { PassThrough, Transform } = require('stream');
 const bunyanDebugStream = require('bunyan-debug-stream');
 const duplexify = require('duplexify');
 const fs = require('fs-extra');
-const glob = require('glob');
 const multiSort = require('multi-sort-stream');
 const pipe = require('multipipe');
 const JsonlParser = require('stream-json/jsonl/Parser');
 const stripAnsi = require('strip-ansi');
 const { AbstractEventBuilder } = require('trace-event-lib');
 
-const temporary = require('../../artifacts/utils/temporaryPath');
 const log = require('../../utils/logger').child({ cat: 'logger' });
 
 function compareTimestamps(a, b) {
@@ -143,27 +141,49 @@ class SimpleEventBuilder extends AbstractEventBuilder {
   }
 }
 
-function chromeTraceStream() {
+const ERROR_TID = 37707;
+
+function getTidHash({ pid, tid, cat }) {
+  const mainCategory = String(cat).split(',')[0];
+  return `${pid}:${mainCategory}:${tid}`;
+}
+
+function getFixedTID(tidHashMap, event) {
+  let tid = tidHashMap.get(getTidHash(event));
+  if (tid === undefined) {
+    tid = ERROR_TID;
+  }
+
+  return tid;
+}
+
+/**
+ * @param {Map<string, number>} [tidHashMap]
+ */
+function chromeTraceStream(tidHashMap) {
   const knownPids = new Set();
   const knownTids = new Set();
+  const fixTID = tidHashMap ? getFixedTID.bind(null, tidHashMap) : (x) => x.tid;
 
   return flatMapTransform((data) => {
     // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
-    const { cat = 'default', msg: name, ph = 'i', pid, tid, time, name: _name, hostname: _hostname, ...args } = data;
+    const { cat: rawCat, msg: name, ph = 'i', pid, tid: _tid, time, name: _name, hostname: _hostname, ...args } = data;
     const ts = new Date(time).getTime() * 1E3;
+    const cat = rawCat || 'undefined';
+    const tid = fixTID(data);
 
     const builder = new SimpleEventBuilder();
     if (!knownPids.has(pid)) {
-      builder.process_name(pid === process.pid ? 'primary' : 'secondary', pid);
+      builder.metadata({ pid, ts, name: 'process_name', args: { name: pid === process.pid ? 'primary' : 'secondary' } });
+      builder.metadata({ pid, ts, name: 'process_sort_index', args: { sort_index: knownPids.size } });
       knownPids.add(pid);
     }
 
-    const tidHash = `${pid}:${tid}`;
-    if (!knownTids.has(tidHash)) {
+    if (!knownTids.has(tid)) {
       const primaryCategory = cat.split(',', 1)[0];
-      builder.thread_name(primaryCategory, tid, pid);
-      builder.thread_sort_index(tid, tid, pid);
-      knownTids.add(tidHash);
+      builder.metadata({ pid, tid, ts, name: 'thread_name', args: { name: primaryCategory } });
+      builder.metadata({ pid, tid, ts, name: 'thread_sort_index', args: { sort_index: tid } });
+      knownTids.add(tid);
     }
 
     const event = { ph, name, pid, tid, cat, ts, args };
@@ -207,25 +227,13 @@ function readJSONL() {
 }
 
 /**
- * @param {string} sessionId
+ * @param {string[]} logs
  * @returns {NodeJS.ReadableStream}
  */
-function uniteSessionLogs(sessionId) {
+function uniteSessionLogs(logs) {
   const readable = through();
-
-  glob(temporary.for.jsonl(`${sessionId}.*`), function (err, logs) {
-    if (err) {
-      return readable.emit('error', err);
-    }
-
-    if (logs.length === 0) {
-      return;
-    }
-
-    const jsonlStreams = logs.map(filePath => fs.createReadStream(filePath).pipe(readJSONL()));
-    mergeSortedJSONL(jsonlStreams).pipe(readable);
-  });
-
+  const jsonlStreams = logs.map(filePath => fs.createReadStream(filePath).pipe(readJSONL()));
+  mergeSortedJSONL(jsonlStreams).pipe(readable);
   return readable;
 }
 

package/src/logger/utils/tracerLegacy.js

@@ -1,45 +1,34 @@
 const methods = {
   startSection(logger) {
-    return (msg, args) => logger.trace.begin(args, msg);
+    return (msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.begin(args, msg);
+      } else {
+        return logger.trace.begin(msg);
+      }
+    };
   },
 
   endSection(logger) {
-    return (msg, args) => logger.trace.end(args);
+    return (_msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.end(args);
+      } else {
+        return logger.trace.end();
+      }
+    };
   },
 
   traceCall(logger) {
     return (name, action, args = {}) => logger.trace.complete(args, name, action);
   },
-
-  invocationCall(logger) {
-    return (sectionName, invocation, action) => {
-      return logger.trace.complete({
-        cat: 'ws-client,ws-client-invocation',
-        data: invocation,
-        stack: _getCallStackTrace(),
-      }, sectionName, action);
-    };
-  },
 };
 
-function _getCallStackTrace() {
-  return new Error().stack
-    .split('\n')
-    .slice(1) // Ignore Error message
-    .map(line => line
-      .replace(/^\s*at\s+/, '')
-      .replace(process.cwd(), '')
-    )
-    .filter(line => !line.includes('/detox/src')) // Ignore detox internal calls
-    .join('\n');
-}
-
 function installLegacyTracerInterface(logger, target) {
   target.traceCall = methods.traceCall(logger);
   target.trace = Object.freeze({
     startSection: methods.startSection(logger),
     endSection: methods.endSection(logger),
-    invocationCall: methods.invocationCall(logger),
   });
 
   return this;