detox 20.1.0-next-is-hittable-check.0 → 20.1.0

package/src/logger/utils/streams/ChromeTraceTransformer.js

@@ -0,0 +1,132 @@
+const { AbstractEventBuilder } = require('trace-event-lib');
+
+const getMainCategory = require('../getMainCategory');
+
+const JSONLStringer = require('./JSONLStringer');
+const { flatMapTransform } = require('./transformers');
+
+class ChromeTraceTransformer {
+  constructor() {
+    /** @type {Map<string, number>} */
+    this._globalThreadMap = null;
+  }
+
+  /**
+   * @param {NodeJS.ReadableStream} eventStream
+   */
+  async scanThreadIDs(eventStream) {
+    const processes = await new Promise((resolve, reject) => {
+      const result = {};
+      eventStream
+        .on('end', () => resolve(result))
+        .on('error', /* istanbul ignore next */ (err) => reject(err))
+        .on('data', (event) => {
+          const { ph, pid, tid, cat } = event;
+          if (ph === 'B' || ph === 'i') {
+            const categories = (result[pid] = result[pid] || {});
+            const mainCategory = String(cat).split(',')[0];
+            const tids = (categories[mainCategory] = categories[mainCategory] || []);
+            if (!tids.includes(tid)) {
+              tids.push(tid);
+            }
+          }
+        });
+    });
+
+    const tidArray = Object.entries(processes).flatMap(([pid, categories]) => {
+      return Object.entries(categories).flatMap(([category, tids]) => {
+        return tids.map(tid => `${pid}:${category}:${tid}`);
+      });
+    });
+
+    this._globalThreadMap = new Map(tidArray.map((hash, index) => [hash, index]));
+  }
+
+  /**
+   * @returns {module:stream.internal.Transform}
+   */
+  createStream() {
+    const transformFn = this._transformBunyanRecord.bind(this, {
+      primaryPid: NaN,
+      knownPids: new Set(),
+      knownTids: new Set(),
+    });
+
+    return flatMapTransform(transformFn);
+  }
+
+  createSerializedStream() {
+    const writable = this.createStream();
+    const readable = JSONLStringer.serializeJSON();
+    writable.pipe(readable);
+    return { writable, readable };
+  }
+
+  /**
+   * @param {{ knownPids: Set<number>; knownTids: Set<number>; primaryPid: number; }} state
+   * @param {*} bunyanLogRecord
+   * @returns {import('trace-event-lib').Event[]}
+   * @private
+   */
+  _transformBunyanRecord(state, bunyanLogRecord) {
+    // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
+    const { cat: rawCat, msg: name, ph, pid, tid: _tid, time, name: _name, hostname: _hostname, ...args } = bunyanLogRecord;
+    const ts = new Date(time).getTime() * 1E3;
+    const cat = rawCat || 'undefined';
+    const tid = this._resolveGlobalTID(bunyanLogRecord);
+
+    const builder = new SimpleEventBuilder();
+    if (!state.knownPids.has(pid)) {
+      if (Number.isNaN(state.primaryPid)) {
+        state.primaryPid = pid;
+      }
+
+      builder.metadata({ pid, ts, name: 'process_name', args: { name: pid === state.primaryPid ? 'primary' : 'secondary' } });
+      builder.metadata({ pid, ts, name: 'process_sort_index', args: { sort_index: state.knownPids.size } });
+      state.knownPids.add(pid);
+    }
+
+    if (!state.knownTids.has(tid)) {
+      const mainCategory = getMainCategory(cat);
+      builder.metadata({ pid, tid, ts, name: 'thread_name', args: { name: mainCategory } });
+      builder.metadata({ pid, tid, ts, name: 'thread_sort_index', args: { sort_index: tid } });
+      state.knownTids.add(tid);
+    }
+
+    const event = { ph, name, pid, tid, cat, ts, args };
+    if (ph === 'E') {
+      delete event.name;
+    }
+
+    builder.events.push(event);
+    return builder.events;
+  }
+
+  _resolveGlobalTID(event) {
+    const hash = this._computeThreadHash(event);
+    const tid = this._globalThreadMap ? this._globalThreadMap.get(hash) : event.tid;
+
+    // Impossible condition, but anyway it is better to be safe than sorry.
+    /* istanbul ignore next */
+    return tid === undefined ? ChromeTraceTransformer.ERROR_TID : tid;
+  }
+
+  _computeThreadHash({ pid, tid, cat }) {
+    return `${pid}:${getMainCategory(cat)}:${tid}`;
+  }
+
+  /* istanbul ignore next */
+  static get ERROR_TID() {
+    return 37707;
+  }
+}
+
+class SimpleEventBuilder extends AbstractEventBuilder {
+  events = [];
+
+  send(event) {
+    this.events.push(event);
+  }
+}
+
+module.exports = ChromeTraceTransformer;

package/src/logger/utils/streams/DetoxJSONLParser.js

@@ -0,0 +1,31 @@
+const JsonlParser = require('stream-json/jsonl/Parser');
+
+const { through } = require('./transformers');
+
+class DetoxJSONLParser {
+  constructor(log) {
+    this._log = log;
+  }
+
+  createTransformer() {
+    const log = this._log;
+    const readable = through();
+    const writable = JsonlParser.make({ checkErrors: true })
+      .on('error', (err) => {
+        /* istanbul ignore else */
+        if (err instanceof SyntaxError) {
+          log.debug({ err });
+          readable.end();
+        } else {
+          readable.emit('error', err);
+        }
+      });
+
+    return {
+      writable,
+      readable: writable.pipe(readable),
+    };
+  }
+}
+
+module.exports = DetoxJSONLParser;

package/src/logger/utils/streams/JSONLStringer.js

@@ -0,0 +1,55 @@
+const { Transform } = require('stream');
+
+class JSONLStringer extends Transform {
+  constructor({ replacer, header, delimiter, footer }) {
+    super({ writableObjectMode: true, readableObjectMode: false });
+
+    this._replacer = replacer;
+    this._header = header;
+    this._delimiter = delimiter;
+    this._footer = footer;
+  }
+
+  _transform(chunk, _, callback) {
+    if (this._header) {
+      this.push(this._header);
+    }
+
+    this.push(JSON.stringify(chunk, this._replacer));
+    this._transform = this._nextTransform;
+    callback(null);
+  }
+
+  _nextTransform(chunk, _, callback) {
+    this.push(this._delimiter + JSON.stringify(chunk, this._replacer));
+    callback(null);
+  }
+
+  _flush(callback) {
+    if (this._footer) {
+      this.push(this._footer);
+    }
+
+    callback();
+  }
+
+  static serializeJSONL() {
+    return new JSONLStringer({
+      replacer: undefined,
+      header: '',
+      delimiter: '\n',
+      footer: '',
+    });
+  }
+
+  static serializeJSON() {
+    return new JSONLStringer({
+      replacer: undefined,
+      header: '[\n\t',
+      delimiter: ',\n\t',
+      footer: '\n]\n',
+    });
+  }
+}
+
+module.exports = JSONLStringer;

package/src/logger/utils/streams/index.js

@@ -0,0 +1,7 @@
+const BunyanTransformer = require('./BunyanTransformer');
+const ChromeTraceTransformer = require('./ChromeTraceTransformer');
+
+module.exports = {
+  BunyanTransformer,
+  ChromeTraceTransformer,
+};

package/src/logger/utils/streams/transformers.js

@@ -0,0 +1,39 @@
+const { PassThrough, Transform } = require('stream');
+
+function through() {
+  return new PassThrough({ objectMode: true });
+}
+
+function mapTransform(fn) {
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback){
+      this.push(fn(chunk));
+      callback();
+    },
+  });
+}
+
+function flatMapTransform(fn) {
+  let index = 0;
+
+  return new Transform({
+    objectMode: true,
+    transform(chunk, encoding, callback){
+      const results = fn(chunk, index++);
+      // eslint-disable-next-line unicorn/no-array-method-this-argument
+      results.forEach(pushThis, this);
+      callback();
+    },
+  });
+}
+
+function pushThis(x) {
+  return this.push(x);
+}
+
+module.exports = {
+  through,
+  mapTransform,
+  flatMapTransform,
+};

package/src/logger/utils/tracerLegacy.js

@@ -1,45 +1,34 @@
 const methods = {
   startSection(logger) {
-    return (msg, args) => logger.trace.begin(args, msg);
+    return (msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.begin(args, msg);
+      } else {
+        return logger.trace.begin(msg);
+      }
+    };
   },
 
   endSection(logger) {
-    return (msg, args) => logger.trace.end(args);
+    return (_msg, args) => {
+      if (args !== undefined) {
+        return logger.trace.end(args);
+      } else {
+        return logger.trace.end();
+      }
+    };
   },
 
   traceCall(logger) {
     return (name, action, args = {}) => logger.trace.complete(args, name, action);
   },
-
-  invocationCall(logger) {
-    return (sectionName, invocation, action) => {
-      return logger.trace.complete({
-        cat: 'ws-client,ws-client-invocation',
-        data: invocation,
-        stack: _getCallStackTrace(),
-      }, sectionName, action);
-    };
-  },
 };
 
-function _getCallStackTrace() {
-  return new Error().stack
-    .split('\n')
-    .slice(1) // Ignore Error message
-    .map(line => line
-      .replace(/^\s*at\s+/, '')
-      .replace(process.cwd(), '')
-    )
-    .filter(line => !line.includes('/detox/src')) // Ignore detox internal calls
-    .join('\n');
-}
-
 function installLegacyTracerInterface(logger, target) {
   target.traceCall = methods.traceCall(logger);
   target.trace = Object.freeze({
     startSection: methods.startSection(logger),
     endSection: methods.endSection(logger),
-    invocationCall: methods.invocationCall(logger),
   });
 
   return this;

package/src/realms/DetoxContext.js

@@ -2,16 +2,19 @@ const funpermaproxy = require('funpermaproxy');
 
 const temporary = require('../artifacts/utils/temporaryPath');
 const { DetoxRuntimeError } = require('../errors');
-const { DetoxLogger, installLegacyTracerInterface } = require('../logger');
+const { DetoxLogger, DetoxLogFinalizer, installLegacyTracerInterface } = require('../logger');
 const symbols = require('../symbols');
 
 const DetoxConstants = require('./DetoxConstants');
 
+//#region Protected symbols
 const $cleanup = Symbol('cleanup');
+const $logFinalizer = Symbol('logFinalizer');
 const $restoreSessionState = Symbol('restoreSessionState');
 const $sessionState = Symbol('restoreSessionState');
 const $status = Symbol('status');
 const $worker = Symbol('worker');
+//#endregion
 
 class DetoxContext {
   constructor() {
@@ -56,6 +59,11 @@ class DetoxContext {
     this.log = this[symbols.logger].child({ cat: 'user' });
     installLegacyTracerInterface(this.log, this);
 
+    this[$logFinalizer] = new DetoxLogFinalizer({
+      session: this[$sessionState],
+      logger: this[symbols.logger],
+    });
+
     /** @type {import('../DetoxWorker') | null} */
     this[$worker] = null;
   }
@@ -89,12 +97,7 @@ class DetoxContext {
   [symbols.config] = funpermaproxy(() => this[symbols.session].detoxConfig);
   [symbols.session] = funpermaproxy(() => this[$sessionState]);
   [symbols.tracing] = Object.freeze({
-    createEventStream: () => {
-      const streamUtils = require('../logger/utils/streamUtils');
-      return streamUtils
-        .uniteSessionLogs(this[$sessionState].id)
-        .pipe(streamUtils.chromeTraceStream());
-    },
+    createEventStream: () => this[$logFinalizer].createEventStream(),
   });
   /** @abstract */
   [symbols.reportTestResults](_testResults) {}
@@ -173,6 +176,7 @@ class DetoxContext {
 module.exports = DetoxContext;
 module.exports.protected = {
   $cleanup,
+  $logFinalizer,
   $restoreSessionState,
   $status,
   $sessionState,

package/src/realms/DetoxInternalsFacade.js

@@ -15,16 +15,10 @@ class DetoxInternalsFacade {
     this.installWorker = context[symbols.installWorker];
     this.uninstallWorker = context[symbols.uninstallWorker];
     this.onHookFailure = context[symbols.onHookFailure];
-    this.onHookStart = context[symbols.onHookStart];
-    this.onHookSuccess = context[symbols.onHookSuccess];
     this.onRunDescribeFinish = context[symbols.onRunDescribeFinish];
     this.onRunDescribeStart = context[symbols.onRunDescribeStart];
-    this.onRunFinish = context[symbols.onRunFinish];
-    this.onRunStart = context[symbols.onRunStart];
     this.onTestDone = context[symbols.onTestDone];
     this.onTestFnFailure = context[symbols.onTestFnFailure];
-    this.onTestFnStart = context[symbols.onTestFnStart];
-    this.onTestFnSuccess = context[symbols.onTestFnSuccess];
     this.onTestStart = context[symbols.onTestStart];
     this.reportTestResults = context[symbols.reportTestResults];
     this.resolveConfig = context[symbols.resolveConfig];

package/src/realms/DetoxPrimaryContext.js

@@ -1,34 +1,31 @@
-const path = require('path');
 const { URL } = require('url');
-const { promisify } = require('util');
 
 const fs = require('fs-extra');
-const glob = require('glob');
-const pipe = require('multipipe');
 const onSignalExit = require('signal-exit');
 
 const temporary = require('../artifacts/utils/temporaryPath');
 const { DetoxRuntimeError } = require('../errors');
 const SessionState = require('../ipc/SessionState');
 const symbols = require('../symbols');
-
-const globAsync = promisify(glob);
-const globSync = glob.sync;
+const { getCurrentCommand } = require('../utils/argparse');
+const uuid = require('../utils/uuid');
 
 const DetoxContext = require('./DetoxContext');
 
-const { $restoreSessionState, $sessionState, $worker } = DetoxContext.protected;
+// Protected symbols
+const { $logFinalizer, $restoreSessionState, $sessionState, $worker } = DetoxContext.protected;
 
-const _finalizeLogs = Symbol('finalizeLogs');
-const _finalizeLogsSync = Symbol('finalizeLogsSync');
+//#region Private symbols
 const _globalLifecycleHandler = Symbol('globalLifecycleHandler');
 const _ipcServer = Symbol('ipcServer');
 const _resetLockFile = Symbol('resetLockFile');
 const _wss = Symbol('wss');
 const _dirty = Symbol('dirty');
 const _emergencyTeardown = Symbol('emergencyTeardown');
-const _areLogsEnabled = Symbol('areLogsEnabled');
 const _lifecycleLogger = Symbol('lifecycleLogger');
+const _sessionFile = Symbol('sessionFile');
+const _logFinalError = Symbol('logFinalError');
+//#endregion
 
 class DetoxPrimaryContext extends DetoxContext {
   constructor() {
@@ -37,6 +34,8 @@ class DetoxPrimaryContext extends DetoxContext {
     this[_dirty] = false;
     this[_wss] = null;
     this[_globalLifecycleHandler] = null;
+    /** Path to file where the initial session object is serialized */
+    this[_sessionFile] = '';
     /**
      * @type {import('../ipc/IPCServer') | null}
      */
@@ -88,9 +87,11 @@ class DetoxPrimaryContext extends DetoxContext {
     await this[symbols.logger].setConfig(loggerConfig);
 
     this[_lifecycleLogger].trace.begin({
+      cwd: process.cwd(),
       data: this[$sessionState],
-    }, process.argv.slice(1).join(' '));
+    }, getCurrentCommand());
 
+    // TODO: IPC Server creation ought to be delegated to a generator/factory.
     const IPCServer = require('../ipc/IPCServer');
     this[_ipcServer] = new IPCServer({
       sessionState: this[$sessionState],
@@ -110,6 +111,7 @@ class DetoxPrimaryContext extends DetoxContext {
       await this[_resetLockFile]();
     }
 
+    // TODO: Detox-server creation ought to be delegated to a generator/factory.
     const DetoxServer = require('../server/DetoxServer');
     if (sessionConfig.autoStart) {
       this[_wss] = new DetoxServer({
@@ -122,15 +124,18 @@ class DetoxPrimaryContext extends DetoxContext {
       await this[_wss].open();
     }
 
+    // TODO: double check that this config is indeed propogated onto the client create at the detox-worker side
     if (!sessionConfig.server && this[_wss]) {
       // @ts-ignore
       sessionConfig.server = `ws://localhost:${this[_wss].port}`;
     }
 
-    await fs.writeFile(this[$sessionState].detoxConfigSnapshotPath, this[$sessionState].stringify());
-    process.env.DETOX_CONFIG_SNAPSHOT_PATH = this[$sessionState].detoxConfigSnapshotPath;
+    this[_sessionFile] = temporary.for.json(this[$sessionState].id);
+    await fs.writeFile(this[_sessionFile], this[$sessionState].stringify());
+    process.env.DETOX_CONFIG_SNAPSHOT_PATH = this[_sessionFile];
+    this[_lifecycleLogger].trace(`Serialized the session state at: ${this[_sessionFile]}`);
 
-    if (opts.workerId !== null) {
+    if (opts.workerId != null) {
       await this[symbols.installWorker](opts);
     }
   }
@@ -140,10 +145,12 @@ class DetoxPrimaryContext extends DetoxContext {
    * @param {Partial<DetoxInternals.DetoxInstallWorkerOptions>} [opts]
    */
   async [symbols.installWorker](opts = {}) {
-    const workerId = this[$sessionState].workerId = opts.workerId = opts.workerId || 'worker';
+    const workerId = opts.workerId || 'worker';
+
+    this[$sessionState].workerId = workerId;
     this[_ipcServer].onRegisterWorker({ workerId });
 
-    await super[symbols.installWorker](opts);
+    await super[symbols.installWorker]({ ...opts, workerId });
   }
 
   /** @override */
@@ -168,14 +175,17 @@ class DetoxPrimaryContext extends DetoxContext {
         this[_ipcServer] = null;
       }
 
-      await fs.remove(this[$sessionState].detoxConfigSnapshotPath);
+      if (this[_sessionFile]) {
+        await fs.remove(this[_sessionFile]);
+      }
 
       if (this[_dirty]) {
         try {
           this[_lifecycleLogger].trace.end();
-          await this[_finalizeLogs]();
+          await this[symbols.logger].close();
+          await this[$logFinalizer].finalize();
         } catch (err) {
-          this[_lifecycleLogger].error({ err }, 'Encountered an error while merging the process logs:');
+          this[_logFinalError](err);
         }
       }
     }
@@ -199,14 +209,23 @@ class DetoxPrimaryContext extends DetoxContext {
       this[_ipcServer].dispose();
     }
 
+    if (this[_sessionFile]) {
+      fs.removeSync(this[_sessionFile]);
+    }
+
     try {
       this[_lifecycleLogger].trace.end({ abortSignal: signal });
-      this[_finalizeLogsSync]();
+      this[symbols.logger].close().catch(this[_logFinalError]);
+      this[$logFinalizer].finalizeSync();
     } catch (err) {
-      this[symbols.logger].error({ err }, 'Encountered an error while merging the process logs:');
+      this[_logFinalError](err);
     }
   };
 
+  [_logFinalError] = (err) => {
+    this[_lifecycleLogger].error(err, 'Encountered an error while merging the process logs:');
+  };
+
   //#endregion
 
   //#region Protected members
@@ -217,77 +236,13 @@ class DetoxPrimaryContext extends DetoxContext {
    */
   [$restoreSessionState]() {
     return new SessionState({
-      detoxConfigSnapshotPath: temporary.for.json(),
+      id: uuid.UUID(),
       detoxIPCServer: `primary-${process.pid}`,
     });
   }
   //#endregion
 
   //#region Private members
-  async[_finalizeLogs]() {
-    const sessionId = this[$sessionState].id;
-    const logs = await globAsync(temporary.for.jsonl(`${sessionId}.*`));
-    if (logs.length === 0) {
-      return;
-    }
-
-    if (this[_areLogsEnabled]()) {
-      const streamUtils = require('../logger/utils/streamUtils');
-      const { rootDir } = this[symbols.config].artifacts;
-
-      await fs.mkdirp(rootDir);
-      const [out1Stream, out2Stream] = ['detox.log', 'detox.trace.json']
-        .map((filename) => fs.createWriteStream(path.join(rootDir, filename)));
-
-      const mergedStream = streamUtils.uniteSessionLogs(sessionId);
-
-      await Promise.all([
-        pipe(mergedStream, streamUtils.debugStream(this[symbols.logger].config.options), out1Stream),
-        pipe(mergedStream, streamUtils.chromeTraceStream(), streamUtils.writeJSON(), out2Stream),
-      ]);
-    }
-
-    await Promise.all(logs.map(filepath => fs.remove(filepath)));
-  }
-
-  async[_finalizeLogsSync]() {
-    const logsEnabled = this[_areLogsEnabled]();
-
-    const { rootDir } = this[symbols.config].artifacts;
-
-    if (logsEnabled) {
-      fs.mkdirpSync(rootDir);
-    }
-
-    const sessionId = this[$sessionState].id;
-    const logs = globSync(temporary.for.jsonl(`${sessionId}.*`));
-
-    for (const log of logs) {
-      if (logsEnabled) {
-        fs.moveSync(log, path.join(rootDir, path.basename(log)));
-      } else {
-        fs.removeSync(log);
-      }
-    }
-  }
-
-  [_areLogsEnabled]() {
-    const { rootDir, plugins } = this[symbols.config].artifacts || {};
-    if (!rootDir || !plugins) {
-      return false;
-    }
-
-    if (!plugins.log.enabled) {
-      return false;
-    }
-
-    if (!plugins.log.keepOnlyFailedTestsArtifacts) {
-      return true;
-    }
-
-    return this[$sessionState].testResults.some(r => !r.success);
-  }
-
   async[_resetLockFile]() {
     const DeviceRegistry = require('../devices/DeviceRegistry');
 

package/src/realms/DetoxSecondaryContext.js

@@ -1,4 +1,4 @@
-const fs = require('fs');
+const fs = require('fs-extra');
 
 const { DetoxInternalError } = require('../errors');
 const SessionState = require('../ipc/SessionState');
@@ -6,9 +6,12 @@ const symbols = require('../symbols');
 
 const DetoxContext = require('./DetoxContext');
 
+// Protected symbols
 const { $restoreSessionState, $sessionState, $worker } = DetoxContext.protected;
+
+//#region Private symbols
 const _ipcClient = Symbol('ipcClient');
-const _shortLifecycle = Symbol('shortLifecycle');
+//#endregion
 
 class DetoxSecondaryContext extends DetoxContext {
   constructor() {
@@ -19,13 +22,6 @@ class DetoxSecondaryContext extends DetoxContext {
      * @type {import('../ipc/IPCClient')}
      */
     this[_ipcClient] = null;
-    /**
-     * @private
-     * @type {undefined | boolean}
-     *
-     * TODO: explain what is short lifecycle and why we need it
-     */
-    this[_shortLifecycle] = false;
   }
 
   //#region Internal members
@@ -47,7 +43,7 @@ class DetoxSecondaryContext extends DetoxContext {
 
     this[_ipcClient] = new IPCClient({
       id: `secondary-${process.pid}`,
-      state: this[$sessionState],
+      sessionState: this[$sessionState],
       logger: this[symbols.logger],
     });
 
@@ -74,9 +70,9 @@ class DetoxSecondaryContext extends DetoxContext {
 
   /** @override */
   async [symbols.installWorker](opts = {}) {
-    const workerId = opts.workerId = opts.workerId || 'worker';
+    const workerId = opts.workerId || 'worker';
     await this[_ipcClient].registerWorker(workerId);
-    await super[symbols.installWorker](opts);
+    await super[symbols.installWorker]({ ...opts, workerId });
   }
   //#endregion