detox 20.0.15-prerelease.0 → 20.1.0

package/src/logger/utils/DetoxLogFinalizer.js

@@ -1,57 +1,62 @@
 const path = require('path');
-const { promisify } = require('util');
 
 const fs = require('fs-extra');
-const glob = require('glob');
-const pipe = require('multipipe');
 
 const temporary = require('../../artifacts/utils/temporaryPath');
 
-const globAsync = promisify(glob);
-const globSync = glob.sync;
-const streamUtils = () => require('../../logger/utils/streamUtils');
+const { BunyanTransformer, ChromeTraceTransformer } = require('./streams');
 
 /**
  * @typedef DetoxLogFinalizerConfig
  * @property {import('../../ipc/SessionState')} session
+ * @property {import('../DetoxLogger')} logger
  */
 
 class DetoxLogFinalizer {
   /** @param {DetoxLogFinalizerConfig} config */
   constructor(config) {
     this._session = config.session;
+    this._bunyanTransformer = new BunyanTransformer(
+      config.logger.child({ cat: 'logger' }),
+    );
+    this._chromeTransformer = new ChromeTraceTransformer();
   }
 
   createEventStream() {
     const sessionId = this._session.id;
-    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+    const logs = temporary.find.jsonl.sync(`${sessionId}.*`);
 
-    return streamUtils()
-      .uniteSessionLogs(logs)
-      .pipe(streamUtils().chromeTraceStream());
+    const toChromeTrace = this._chromeTransformer.createStream();
+    return this._bunyanTransformer.uniteSessionLogs(logs).pipe(toChromeTrace);
   }
 
   async finalize() {
     const sessionId = this._session.id;
-    const logs = await globAsync(temporary.for.jsonl(`${sessionId}.*`));
+    const logs = await temporary.find.jsonl.async(`${sessionId}.*`);
     if (logs.length === 0) {
       return;
     }
 
-    if (this._areLogsEnabled()) {
+    if (this._shouldSaveLogs()) {
       const rootDir = this._config.artifacts.rootDir;
 
       await fs.mkdirp(rootDir);
-      const [out1Stream, out2Stream] = ['detox.log', 'detox.trace.json']
-        .map((filename) => fs.createWriteStream(path.join(rootDir, filename)));
 
-      const tidHashMap = await this._scanForThreadIds(logs);
-      const mergedStream = streamUtils().uniteSessionLogs(logs);
+      const firstPass = this._bunyanTransformer.uniteSessionLogs(logs);
+      await this._chromeTransformer.scanThreadIDs(firstPass);
 
-      await Promise.all([
-        pipe(mergedStream, streamUtils().debugStream(this._config.logger.options), out1Stream),
-        pipe(mergedStream, streamUtils().chromeTraceStream(tidHashMap), streamUtils().writeJSON(), out2Stream),
-      ]);
+      const secondPass = this._bunyanTransformer.uniteSessionLogs(logs);
+      const outStreams = [this._createPlainFileStream(), this._createChromeTraceStream()];
+
+      await Promise.all(outStreams.map(stream => {
+        return new Promise((resolve, reject) => {
+          stream.target
+            .on('finish', resolve)
+            .on('error', reject);
+
+          secondPass.pipe(stream.writable);
+        });
+      }));
     }
 
     await Promise.all(logs.map(filepath => fs.remove(filepath)));
@@ -59,26 +64,72 @@ class DetoxLogFinalizer {
 
   finalizeSync() {
     const sessionId = this._session.id;
-    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
+    const logs = temporary.find.jsonl.sync(`${sessionId}.*`);
     if (logs.length === 0) {
       return;
     }
 
-    const logsEnabled = this._areLogsEnabled();
-    const rootDir = logsEnabled ? this._config.artifacts.rootDir : '';
-    if (logsEnabled) {
+    const shouldSaveLogs = this._shouldSaveLogs(true);
+    const rootDir = shouldSaveLogs ? this._config.artifacts.rootDir : '';
+    if (shouldSaveLogs) {
       fs.mkdirpSync(rootDir);
     }
 
     for (const log of logs) {
-      if (logsEnabled) {
-        fs.moveSync(log, path.join(rootDir, path.basename(log)));
+      if (shouldSaveLogs) {
+        const dest = path.join(rootDir, path.basename(log));
+        this._safeMoveSync(log, dest);
       } else {
-        fs.removeSync(log);
+        this._safeRemoveSync(log);
+      }
+    }
+  }
+
+  _createPlainFileStream() {
+    const rootDir = this._config.artifacts.rootDir;
+    const bunyanOptions = this._config.logger.options;
+    const transformer = this._bunyanTransformer.createPlainTransformer(bunyanOptions);
+    const fileStream = fs.createWriteStream(path.join(rootDir, 'detox.log'));
+    transformer.readable.pipe(fileStream);
+
+    return { writable: transformer.writable, target: fileStream };
+  }
+
+  _createChromeTraceStream() {
+    const rootDir = this._config.artifacts.rootDir;
+    const transformer = this._chromeTransformer.createSerializedStream();
+    const fileStream = fs.createWriteStream(path.join(rootDir, 'detox.trace.json'));
+    transformer.readable.pipe(fileStream);
+
+    return { writable: transformer.writable, target: fileStream };
+  }
+
+  /* istanbul ignore next */
+  _safeMoveSync(src, dest) {
+    // Using console.* instead of logger.* because this is the end of the execution,
+    // and the logger is already closed.
+
+    try {
+      fs.moveSync(src, dest);
+    } catch (moveError) {
+      console.warn(`Failed to move a log file from: ${src}.\nReason: ${moveError.message}`);
+      try {
+        fs.copySync(src, dest);
+      } catch (copyError) {
+        console.warn(`Attempt to copy the file also failed.\nReason: ${copyError.message}`);
       }
     }
   }
 
+  /* istanbul ignore next */
+  _safeRemoveSync(filepath) {
+    try {
+      fs.removeSync(filepath);
+    } catch (removeError) {
+      console.warn(`Failed to remove a log file at: ${filepath}.\nReason: ${removeError.message}`);
+    }
+  }
+
   /** @private */
   get _config() {
     // The config appears later in the lifecycle, so we need to access it lazily
@@ -86,7 +137,7 @@ class DetoxLogFinalizer {
   }
 
   /** @private */
-  _areLogsEnabled() {
+  _shouldSaveLogs(isEmergencyExit = false) {
     if (!this._config) {
       return false;
     }
@@ -104,36 +155,11 @@ class DetoxLogFinalizer {
       return true;
     }
 
-    return this._session.testResults.some(r => !r.success);
-  }
-
-  /** @private */
-  async _scanForThreadIds(logs) {
-    const processes = await new Promise((resolve, reject) => {
-      const result = {};
-      streamUtils().uniteSessionLogs(logs)
-        .on('end', () => resolve(result))
-        .on('error', /* istanbul ignore next */ (err) => reject(err))
-        .on('data', (event) => {
-          const { ph, pid, tid, cat } = event;
-          if (ph === 'B' || ph === 'i') {
-            const categories = (result[pid] = result[pid] || {});
-            const mainCategory = String(cat).split(',')[0];
-            const tids = (categories[mainCategory] = categories[mainCategory] || []);
-            if (!tids.includes(tid)) {
-              tids.push(tid);
-            }
-          }
-        });
-    });
-
-    const tidArray = Object.entries(processes).flatMap(([pid, categories]) => {
-      return Object.entries(categories).flatMap(([category, tids]) => {
-        return tids.map(tid => `${pid}:${category}:${tid}`);
-      });
-    });
+    if (isEmergencyExit) {
+      return true;
+    }
 
-    return new Map(tidArray.map((hash, index) => [hash, index]));
+    return this._session.testResults.some(r => !r.success);
   }
 }
 

package/src/logger/utils/MessageStack.js

@@ -1,24 +1,35 @@
+const getMainCategory = require('./getMainCategory');
+
 class MessageStack {
   constructor() {
     this._map = {};
   }
 
-  push(tid, msg) {
-    if (this._map[tid] == null) {
-      this._map[tid] = [];
+  push(context, msg) {
+    const hash = this._hash(context);
+
+    if (this._map[hash] == null) {
+      this._map[hash] = [];
     }
 
-    return this._map[tid].push(msg);
+    return this._map[hash].push(msg);
   }
 
-  pop(tid) {
-    const stack = this._map[tid];
+  pop(context) {
+    const hash = this._hash(context);
+    const stack = this._map[hash];
     if (stack == null || stack.length === 0) {
       return ['<no begin message>'];
     }
 
     return stack.pop();
   }
+
+  _hash(context) {
+    const cat = getMainCategory(context.cat);
+    const tid = context.tid;
+    return `${cat}:${tid}`;
+  }
 }
 
 module.exports = MessageStack;

package/src/logger/utils/customConsoleLogger.js

@@ -47,6 +47,16 @@ function overrideConsoleMethods(console, bunyanLogger) {
   if (!console.__detox_log__) {
     const log = bunyanLogger;
 
+    console.__detox_log__ = {
+      log: console.log,
+      info: console.info,
+      warn: console.warn,
+      trace: console.trace,
+      error: console.error,
+      debug: console.debug,
+      assert: console.assert,
+    };
+
     override(console, 'log', log.info.bind(log));
     override(console, 'info', log.info.bind(log));
     override(console, 'warn', log.warn.bind(log));
@@ -54,13 +64,19 @@ function overrideConsoleMethods(console, bunyanLogger) {
     override(console, 'error', log.error.bind(log));
     override(console, 'debug', log.debug.bind(log));
     override(console, 'assert', log.error.bind(log));
-
-    console.__detox_log__ = log;
   }
 
   return console;
 }
 
+function restoreConsoleMethods(console) {
+  if (console.__detox_log__) {
+    Object.assign(console, console.__detox_log__);
+    delete console.__detox_log__;
+  }
+}
+
 module.exports = {
   overrideConsoleMethods,
+  restoreConsoleMethods,
 };

package/src/logger/utils/getMainCategory.js

@@ -0,0 +1,5 @@
+function getMainCategory(category) {
+  return category ? String(category).split(',', 1)[0] : 'undefined';
+}
+
+module.exports = getMainCategory;

package/src/logger/utils/streams/BunyanTransformer.js

@@ -0,0 +1,72 @@
+const fs = require('fs');
+const { PassThrough } = require('stream');
+
+const bunyanDebugStream = require('bunyan-debug-stream');
+const multiSort = require('multi-sort-stream');
+const pipe = require('multipipe');
+const stripAnsi = require('strip-ansi');
+
+const DetoxJSONLParser = require('./DetoxJSONLParser');
+const { mapTransform, through } = require('./transformers');
+
+class BunyanTransformer {
+  /**
+   * @param {Detox.Logger} log
+   */
+  constructor(log) {
+    this._jsonlParser = new DetoxJSONLParser(log);
+  }
+
+  /**
+   * @param {string[]} logFilePaths
+   */
+  uniteSessionLogs(logFilePaths) {
+    const intermediate = mapTransform(BunyanTransformer.normalizeBunyanRecord);
+    const reemitError = (err) => intermediate.emit('error', err);
+
+    const jsonlStreams = logFilePaths.map(filePath => {
+      const { readable, writable } = this._jsonlParser.createTransformer();
+      fs.createReadStream(filePath)
+        .on('error', reemitError)
+        .pipe(writable);
+
+      return readable;
+    });
+
+    const multisorted = multiSort(jsonlStreams, BunyanTransformer.compareTimestamps);
+    return pipe(multisorted, intermediate, through());
+  }
+
+  createPlainTransformer(bunyanOptions) {
+    /** @type {*} */
+    const readable = new PassThrough({ encoding: 'utf8', objectMode: false });
+    const writable = bunyanDebugStream.default({
+      ...bunyanOptions,
+
+      colors: false,
+      out: readable,
+    });
+
+    writable.on('error', /* istanbul ignore next */ (err) => readable.emit('error', err));
+    writable.on('finish', () => readable.end());
+
+    return {
+      writable,
+      readable,
+    };
+  }
+
+  static normalizeBunyanRecord(record) {
+    const value = record.value;
+    value.msg = stripAnsi(value.msg);
+    value.time = new Date(value.time);
+
+    return value;
+  }
+
+  static compareTimestamps(a, b) {
+    return +(a.value.time > b.value.time) - +(a.value.time < b.value.time);
+  }
+}
+
+module.exports = BunyanTransformer;

package/src/logger/utils/streams/ChromeTraceTransformer.js

@@ -0,0 +1,132 @@
+const { AbstractEventBuilder } = require('trace-event-lib');
+
+const getMainCategory = require('../getMainCategory');
+
+const JSONLStringer = require('./JSONLStringer');
+const { flatMapTransform } = require('./transformers');
+
+class ChromeTraceTransformer {
+  constructor() {
+    /** @type {Map<string, number>} */
+    this._globalThreadMap = null;
+  }
+
+  /**
+   * @param {NodeJS.ReadableStream} eventStream
+   */
+  async scanThreadIDs(eventStream) {
+    const processes = await new Promise((resolve, reject) => {
+      const result = {};
+      eventStream
+        .on('end', () => resolve(result))
+        .on('error', /* istanbul ignore next */ (err) => reject(err))
+        .on('data', (event) => {
+          const { ph, pid, tid, cat } = event;
+          if (ph === 'B' || ph === 'i') {
+            const categories = (result[pid] = result[pid] || {});
+            const mainCategory = String(cat).split(',')[0];
+            const tids = (categories[mainCategory] = categories[mainCategory] || []);
+            if (!tids.includes(tid)) {
+              tids.push(tid);
+            }
+          }
+        });
+    });
+
+    const tidArray = Object.entries(processes).flatMap(([pid, categories]) => {
+      return Object.entries(categories).flatMap(([category, tids]) => {
+        return tids.map(tid => `${pid}:${category}:${tid}`);
+      });
+    });
+
+    this._globalThreadMap = new Map(tidArray.map((hash, index) => [hash, index]));
+  }
+
+  /**
+   * @returns {module:stream.internal.Transform}
+   */
+  createStream() {
+    const transformFn = this._transformBunyanRecord.bind(this, {
+      primaryPid: NaN,
+      knownPids: new Set(),
+      knownTids: new Set(),
+    });
+
+    return flatMapTransform(transformFn);
+  }
+
+  createSerializedStream() {
+    const writable = this.createStream();
+    const readable = JSONLStringer.serializeJSON();
+    writable.pipe(readable);
+    return { writable, readable };
+  }
+
+  /**
+   * @param {{ knownPids: Set<number>; knownTids: Set<number>; primaryPid: number; }} state
+   * @param {*} bunyanLogRecord
+   * @returns {import('trace-event-lib').Event[]}
+   * @private
+   */
+  _transformBunyanRecord(state, bunyanLogRecord) {
+    // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+    const { cat: rawCat, msg: name, ph, pid, tid: _tid, time, name: _name, hostname: _hostname, ...args } = bunyanLogRecord;
+    const ts = new Date(time).getTime() * 1E3;
+    const cat = rawCat || 'undefined';
+    const tid = this._resolveGlobalTID(bunyanLogRecord);
+
+    const builder = new SimpleEventBuilder();
+    if (!state.knownPids.has(pid)) {
+      if (Number.isNaN(state.primaryPid)) {
+        state.primaryPid = pid;
+      }
+
+      builder.metadata({ pid, ts, name: 'process_name', args: { name: pid === state.primaryPid ? 'primary' : 'secondary' } });
+      builder.metadata({ pid, ts, name: 'process_sort_index', args: { sort_index: state.knownPids.size } });
+      state.knownPids.add(pid);
+    }
+
+    if (!state.knownTids.has(tid)) {
+      const mainCategory = getMainCategory(cat);
+      builder.metadata({ pid, tid, ts, name: 'thread_name', args: { name: mainCategory } });
+      builder.metadata({ pid, tid, ts, name: 'thread_sort_index', args: { sort_index: tid } });
+      state.knownTids.add(tid);
+    }
+
+    const event = { ph, name, pid, tid, cat, ts, args };
+    if (ph === 'E') {
+      delete event.name;
+    }
+
+    builder.events.push(event);
+    return builder.events;
+  }
+
+  _resolveGlobalTID(event) {
+    const hash = this._computeThreadHash(event);
+    const tid = this._globalThreadMap ? this._globalThreadMap.get(hash) : event.tid;
+
+    // Impossible condition, but anyway it is better to be safe than sorry.
+    /* istanbul ignore next */
+    return tid === undefined ? ChromeTraceTransformer.ERROR_TID : tid;
+  }
+
+  _computeThreadHash({ pid, tid, cat }) {
+    return `${pid}:${getMainCategory(cat)}:${tid}`;
+  }
+
+  /* istanbul ignore next */
+  static get ERROR_TID() {
+    return 37707;
+  }
+}
+
+class SimpleEventBuilder extends AbstractEventBuilder {
+  events = [];
+
+  send(event) {
+    this.events.push(event);
+  }
+}
+
+module.exports = ChromeTraceTransformer;

package/src/logger/utils/streams/DetoxJSONLParser.js

@@ -0,0 +1,31 @@
+const JsonlParser = require('stream-json/jsonl/Parser');
+
+const { through } = require('./transformers');
+
+class DetoxJSONLParser {
+  constructor(log) {
+    this._log = log;
+  }
+
+  createTransformer() {
+    const log = this._log;
+    const readable = through();
+    const writable = JsonlParser.make({ checkErrors: true })
+      .on('error', (err) => {
+        /* istanbul ignore else */
+        if (err instanceof SyntaxError) {
+          log.debug({ err });
+          readable.end();
+        } else {
+          readable.emit('error', err);
+        }
+      });
+
+    return {
+      writable,
+      readable: writable.pipe(readable),
+    };
+  }
+}
+
+module.exports = DetoxJSONLParser;

package/src/logger/utils/streams/JSONLStringer.js

@@ -0,0 +1,55 @@
+const { Transform } = require('stream');
+
+class JSONLStringer extends Transform {
+  constructor({ replacer, header, delimiter, footer }) {
+    super({ writableObjectMode: true, readableObjectMode: false });
+
+    this._replacer = replacer;
+    this._header = header;
+    this._delimiter = delimiter;
+    this._footer = footer;
+  }
+
+  _transform(chunk, _, callback) {
+    if (this._header) {
+      this.push(this._header);
+    }
+
+    this.push(JSON.stringify(chunk, this._replacer));
+    this._transform = this._nextTransform;
+    callback(null);
+  }
+
+  _nextTransform(chunk, _, callback) {
+    this.push(this._delimiter + JSON.stringify(chunk, this._replacer));
+    callback(null);
+  }
+
+  _flush(callback) {
+    if (this._footer) {
+      this.push(this._footer);
+    }
+
+    callback();
+  }
+
+  static serializeJSONL() {
+    return new JSONLStringer({
+      replacer: undefined,
+      header: '',
+      delimiter: '\n',
+      footer: '',
+    });
+  }
+
+  static serializeJSON() {
+    return new JSONLStringer({
+      replacer: undefined,
+      header: '[\n\t',
+      delimiter: ',\n\t',
+      footer: '\n]\n',
+    });
+  }
+}
+
+module.exports = JSONLStringer;

package/src/logger/utils/streams/index.js

@@ -0,0 +1,7 @@
+const BunyanTransformer = require('./BunyanTransformer');
+const ChromeTraceTransformer = require('./ChromeTraceTransformer');
+
+module.exports = {
+  BunyanTransformer,
+  ChromeTraceTransformer,
+};

package/src/logger/utils/streams/transformers.js

@@ -0,0 +1,39 @@
+const { PassThrough, Transform } = require('stream');
+
+function through() {
+  return new PassThrough({ objectMode: true });
+}
+
+function mapTransform(fn) {
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback){
+      this.push(fn(chunk));
+      callback();
+    },
+  });
+}
+
+function flatMapTransform(fn) {
+  let index = 0;
+
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback){
+      const results = fn(chunk, index++);
+      // eslint-disable-next-line unicorn/no-array-method-this-argument
+      results.forEach(pushThis, this);
+      callback();
+    },
+  });
+}
+
+function pushThis(x) {
+  return this.push(x);
+}
+
+module.exports = {
+  through,
+  mapTransform,
+  flatMapTransform,
+};

package/src/realms/DetoxContext.js

@@ -60,7 +60,8 @@ class DetoxContext {
     installLegacyTracerInterface(this.log, this);
 
     this[$logFinalizer] = new DetoxLogFinalizer({
-      session: this[$sessionState]
+      session: this[$sessionState],
+      logger: this[symbols.logger],
     });
 
     /** @type {import('../DetoxWorker') | null} */