@mongosh/node-runtime-worker-thread 2.3.0 → 2.3.2

package/src/index.ts

@@ -1,8 +1,6 @@
 /* istanbul ignore file */
-/* ^^^ we test the dist directly, so isntanbul can't calculate the coverage correctly */
+/* ^^^ we test the dist directly, so istanbul can't calculate the coverage correctly */
 
-import type { ChildProcess, SpawnOptionsWithoutStdio } from 'child_process';
-import { spawn } from 'child_process';
 import type {
   Runtime,
   RuntimeEvaluationListener,
@@ -11,184 +9,162 @@ import type {
 import type { MongoshBus } from '@mongosh/types';
 import path from 'path';
 import { EventEmitter, once } from 'events';
-import { kill } from './spawn-child-from-source';
+import { pathToFileURL } from 'url';
 import type { Caller } from './rpc';
-import { createCaller, cancel } from './rpc';
-import { ChildProcessEvaluationListener } from './child-process-evaluation-listener';
+import { createCaller, cancel, exposeAll } from './rpc';
 import type { WorkerRuntime as WorkerThreadWorkerRuntime } from './worker-runtime';
 import {
   deserializeEvaluationResult,
   serializeConnectOptions,
 } from './serializer';
-import { ChildProcessMongoshBus } from './child-process-mongosh-bus';
 import type { CompassServiceProvider } from '@mongosh/service-provider-server';
+import type { InterruptHandle } from 'interruptor';
+import { interrupt as nativeInterrupt } from 'interruptor';
+import { WorkerThreadEvaluationListener } from './worker-thread-evaluation-listener';
+import { WorkerProcessMongoshBus } from './worker-process-mongosh-bus';
 
 type DevtoolsConnectOptions = Parameters<
   (typeof CompassServiceProvider)['connect']
 >[1];
-type ChildProcessRuntime = Caller<WorkerThreadWorkerRuntime>;
-
-function parseStderrToError(str: string): Error | null {
-  const [, errorMessageWithStack] = str
-    .split(/^\s*\^\s*$/m)
-    .map((part) => part.trim());
-
-  if (errorMessageWithStack) {
-    const e = new Error();
-    const errorHeader =
-      errorMessageWithStack.substring(
-        0,
-        errorMessageWithStack.search(/^\s*at/m)
-      ) || errorMessageWithStack;
-
-    const [name, ...message] = errorHeader.split(': ');
-
-    e.name = name;
-    e.message = message.join(': ').trim();
-    e.stack = errorMessageWithStack;
-
-    return e;
-  }
-
-  return null;
-}
+type WorkerThreadRuntime = Caller<WorkerThreadWorkerRuntime>;
 
 class WorkerRuntime implements Runtime {
   private initOptions: {
     uri: string;
     driverOptions: DevtoolsConnectOptions;
     cliOptions: { nodb?: boolean };
-    spawnOptions: SpawnOptionsWithoutStdio;
+    workerOptions: WorkerOptions;
   };
 
   evaluationListener: RuntimeEvaluationListener | null = null;
 
   private eventEmitter: MongoshBus;
 
-  private childProcessMongoshBus!: ChildProcessMongoshBus;
+  private workerProcess!: Worker;
 
-  private childProcessEvaluationListener!: ChildProcessEvaluationListener;
+  private workerProcessRuntime!: WorkerThreadRuntime;
 
-  private childProcess!: ChildProcess;
+  private initWorkerPromise: Promise<void>;
 
-  private childProcessRuntime!: ChildProcessRuntime;
+  private workerThreadEvaluationListener!: WorkerThreadEvaluationListener;
 
-  private initWorkerPromise: Promise<void>;
+  private workerProcessMongoshBus!: WorkerProcessMongoshBus;
 
-  private childProcessProxySrcPath: string =
-    process.env
-      .CHILD_PROCESS_PROXY_SRC_PATH_DO_NOT_USE_THIS_EXCEPT_FOR_TESTING ||
-    path.resolve(__dirname, 'child-process-proxy.js');
+  private workerProcessPath = path.resolve(__dirname, 'worker-runtime.js');
 
   constructor(
     uri: string,
     driverOptions: DevtoolsConnectOptions,
     cliOptions: { nodb?: boolean } = {},
-    spawnOptions: SpawnOptionsWithoutStdio = {},
+    workerOptions: WorkerOptions = {},
     eventEmitter: MongoshBus = new EventEmitter()
   ) {
-    this.initOptions = { uri, driverOptions, cliOptions, spawnOptions };
+    this.initOptions = { uri, driverOptions, cliOptions, workerOptions };
     this.eventEmitter = eventEmitter;
     this.initWorkerPromise = this.initWorker();
   }
 
   private async initWorker() {
-    const { uri, driverOptions, cliOptions, spawnOptions } = this.initOptions;
-
-    this.childProcess = spawn(
-      process.execPath,
-      [this.childProcessProxySrcPath],
-      {
-        stdio: ['inherit', 'inherit', 'pipe', 'ipc'],
-        ...spawnOptions,
-      }
+    const workerProcess = new Worker(
+      pathToFileURL(this.workerProcessPath).href,
+      this.initOptions.workerOptions
     );
 
-    const waitForReadyMessage = async () => {
-      let msg: string;
-      while (([msg] = await once(this.childProcess, 'message'))) {
-        if (msg === 'ready') return;
-      }
-    };
-
-    let spawnError = '';
+    const workerReadyPromise = async (): Promise<void> => {
+      const waitForReadyMessage = async () => {
+        let msg: {
+          data: string;
+        };
+        while (([msg] = await once(workerProcess, 'message'))) {
+          if (msg?.data === 'ready') return;
+        }
+      };
+
+      const waitForError = async () => {
+        const [err] = await once(workerProcess, 'error');
+        if (err) {
+          err.message = `Worker thread failed to start with error: ${
+            (err as Error).message
+          }`;
+          throw err;
+        }
+      };
 
-    this.childProcess?.stderr?.setEncoding('utf8')?.on('data', (chunk) => {
-      spawnError += chunk;
-    });
+      await Promise.race([waitForReadyMessage(), waitForError()]);
+    };
 
-    const waitForError = async () => {
-      const [exitCode] = await once(this.childProcess, 'exit');
+    await workerReadyPromise();
 
-      if (exitCode) {
-        let error = parseStderrToError(spawnError);
+    const { interrupt } = createCaller(['interrupt'], workerProcess);
 
-        if (error) {
-          error.message = `Child process failed to start with the following error: ${error.message}`;
-        } else {
-          error = new Error(
-            `Worker runtime failed to start: child process exited with code ${
-              exitCode as number | string
-            }`
-          );
-        }
+    let interruptHandle: InterruptHandle | null = null;
 
-        throw error;
+    this.workerProcessRuntime = Object.assign(
+      createCaller(
+        [
+          'init',
+          'evaluate',
+          'getCompletions',
+          'getShellPrompt',
+          'setEvaluationListener',
+          'interrupt',
+        ],
+        workerProcess
+      ),
+      {
+        interrupt(): boolean {
+          if (interruptHandle) {
+            nativeInterrupt(interruptHandle);
+            return true;
+          }
+
+          return interrupt();
+        },
       }
-    };
-
-    await Promise.race([waitForReadyMessage(), waitForError()]);
-
-    // We expect the amount of listeners to be more than the default value of 10
-    // but probably not more than ~25 (all exposed methods on
-    // ChildProcessEvaluationListener and ChildProcessMongoshBus + any
-    // concurrent in-flight calls on ChildProcessRuntime) at once
-    this.childProcess.setMaxListeners(25);
-
-    this.childProcessRuntime = createCaller(
-      [
-        'init',
-        'evaluate',
-        'getCompletions',
-        'setEvaluationListener',
-        'getShellPrompt',
-        'interrupt',
-      ],
-      this.childProcess
     );
 
-    this.childProcessEvaluationListener = new ChildProcessEvaluationListener(
+    this.workerThreadEvaluationListener = new WorkerThreadEvaluationListener(
       this,
-      this.childProcess
+      workerProcess
+    );
+
+    exposeAll(
+      {
+        onRunInterruptible(handle: InterruptHandle | null) {
+          interruptHandle = handle;
+        },
+      },
+      workerProcess
     );
 
-    this.childProcessMongoshBus = new ChildProcessMongoshBus(
+    this.workerProcessMongoshBus = new WorkerProcessMongoshBus(
       this.eventEmitter,
-      this.childProcess
+      workerProcess
     );
 
-    await this.childProcessRuntime.init(
-      uri,
-      serializeConnectOptions(driverOptions),
-      cliOptions
+    await this.workerProcessRuntime.init(
+      this.initOptions.uri,
+      serializeConnectOptions(this.initOptions.driverOptions),
+      this.initOptions.cliOptions
     );
+    this.workerProcess = workerProcess;
   }
 
   async evaluate(code: string): Promise<RuntimeEvaluationResult> {
     await this.initWorkerPromise;
     return deserializeEvaluationResult(
-      await this.childProcessRuntime.evaluate(code)
+      await this.workerProcessRuntime.evaluate(code)
     );
   }
 
   async getCompletions(code: string) {
     await this.initWorkerPromise;
-    return await this.childProcessRuntime.getCompletions(code);
+    return await this.workerProcessRuntime.getCompletions(code);
   }
 
   async getShellPrompt() {
     await this.initWorkerPromise;
-    return await this.childProcessRuntime.getShellPrompt();
+    return await this.workerProcessRuntime.getShellPrompt();
   }
 
   setEvaluationListener(listener: RuntimeEvaluationListener | null) {
@@ -201,28 +177,30 @@ class WorkerRuntime implements Runtime {
     try {
       await this.initWorkerPromise;
     } catch {
-      // In case child process encountered an error during init we still want
-      // to clean up whatever possible
+      // In case the worker thread encountered an error during init
+      // we still want to clean up whatever possible.
     }
 
-    await kill(this.childProcess);
+    if (this.workerProcessRuntime) {
+      this.workerProcessRuntime[cancel]();
+    }
 
-    if (this.childProcessRuntime) {
-      this.childProcessRuntime[cancel]();
+    if (this.workerProcess) {
+      this.workerProcess.terminate();
     }
 
-    if (this.childProcessEvaluationListener) {
-      this.childProcessEvaluationListener.terminate();
+    if (this.workerThreadEvaluationListener) {
+      this.workerThreadEvaluationListener.terminate();
     }
 
-    if (this.childProcessMongoshBus) {
-      this.childProcessMongoshBus.terminate();
+    if (this.workerProcessMongoshBus) {
+      this.workerProcessMongoshBus.terminate();
     }
   }
 
   async interrupt() {
     await this.initWorkerPromise;
-    return this.childProcessRuntime.interrupt();
+    return this.workerProcessRuntime.interrupt();
   }
 
   async waitForRuntimeToBeReady() {

package/src/lock.spec.ts

@@ -25,7 +25,7 @@ describe('Lock', function () {
     // eslint-disable-next-line @typescript-eslint/no-floating-promises
     lock.lock();
 
-    let err: Error;
+    let err!: Error;
 
     try {
       // eslint-disable-next-line @typescript-eslint/no-floating-promises

package/src/rpc.spec.ts

@@ -1,58 +1,24 @@
 import { expect } from 'chai';
 import { EventEmitter } from 'events';
 
-import type { Caller, Exposed } from './rpc';
-import {
-  createCaller,
-  exposeAll,
-  close,
-  cancel,
-  serialize,
-  deserialize,
-  removeTrailingUndefined,
-} from './rpc';
-
-function createMockRpcMesageBus() {
-  const bus = new (class Bus extends EventEmitter {
-    send(data: any) {
-      this.emit('message', data);
-    }
-  })();
-  return bus;
+import type { Caller, Exposed, RPCMessageBus } from './rpc';
+import { createCaller, exposeAll, close, cancel } from './rpc';
+
+function createMockRpcMesageBus(): RPCMessageBus {
+  const ee = new EventEmitter();
+  return {
+    addEventListener: ee.on.bind(ee),
+    removeEventListener: ee.off.bind(ee),
+    postMessage: (data: unknown) => ee.emit('message', data),
+  };
 }
 
 function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-describe('rpc helpers', function () {
-  describe('serialize', function () {
-    it('returns base64 representation of an input', function () {
-      expect(serialize('Hello')).to.match(/data:;base64,\/w[08]iBUhlbGxv/);
-    });
-  });
-
-  describe('deserialize', function () {
-    it("converts base64 representation of input back to it's original form", function () {
-      expect(deserialize(serialize('Hello'))).to.equal('Hello');
-    });
-
-    it("returns original string if it's not a base64 data uri", function () {
-      expect(deserialize('Hi')).to.equal('Hi');
-    });
-  });
-
-  describe('removeTrailingUndefined', function () {
-    it('removes trailing undefineds from an array', function () {
-      expect(
-        removeTrailingUndefined([1, 2, 3, undefined, undefined, undefined])
-      ).to.deep.equal([1, 2, 3]);
-    });
-  });
-});
-
 describe('rpc', function () {
-  let messageBus: EventEmitter;
+  let messageBus: RPCMessageBus | null;
   let caller: Caller<{
     meow(...args: any[]): string;
     throws(...args: any[]): never;
@@ -61,22 +27,21 @@ describe('rpc', function () {
     woof(...args: any[]): string;
     neverResolves(...args: any[]): void;
   }>;
-  let exposed: Exposed<unknown>;
+  let exposed: Exposed<unknown>; // adding `| null` breaks TS type inference
 
   afterEach(function () {
     if (messageBus) {
-      messageBus.removeAllListeners();
       messageBus = null;
     }
 
     if (caller) {
       caller[cancel]();
-      caller = null;
+      caller = null as any;
     }
 
     if (exposed) {
       exposed[close]();
-      exposed = null;
+      exposed = null as any;
     }
   });
 
@@ -109,7 +74,7 @@ describe('rpc', function () {
       messageBus
     );
 
-    let err: Error;
+    let err!: Error;
 
     try {
       // eslint-disable-next-line @typescript-eslint/await-thenable
@@ -126,7 +91,7 @@ describe('rpc', function () {
       .match(/TypeError: Uh-oh, error!\r?\n\s+at throws/);
   });
 
-  it('throws on client if arguments are not serializable', async function () {
+  it('allows undefined response', async function () {
     messageBus = createMockRpcMesageBus();
     caller = createCaller(['callMe'], messageBus);
 
@@ -139,21 +104,11 @@ describe('rpc', function () {
       messageBus
     );
 
-    let err: Error;
-
-    try {
-      await caller.callMe((a: number, b: number) => a + b);
-    } catch (e: any) {
-      err = e;
-    }
-
-    expect(err).to.be.instanceof(Error);
-    expect(err)
-      .to.have.property('message')
-      .match(/could not be cloned/);
+    expect(await caller.callMe((a: number, b: number) => a + b)).to.be
+      .undefined;
   });
 
-  it('throws on client if retured value from the server is not serializable', async function () {
+  it('allows function response', async function () {
     messageBus = createMockRpcMesageBus();
     caller = createCaller(['returnsFunction'], messageBus);
 
@@ -166,18 +121,7 @@ describe('rpc', function () {
       messageBus
     );
 
-    let err: Error;
-
-    try {
-      await caller.returnsFunction();
-    } catch (e: any) {
-      err = e;
-    }
-
-    expect(err).to.be.instanceof(Error);
-    expect(err)
-      .to.have.property('message')
-      .match(/could not be cloned/);
+    expect(await caller.returnsFunction()).to.be.instanceof(Function);
   });
 
   describe('createCaller', function () {
@@ -192,7 +136,7 @@ describe('rpc', function () {
       messageBus = createMockRpcMesageBus();
       caller = createCaller(['meow'], messageBus);
 
-      messageBus.on('message', (data) => {
+      messageBus.addEventListener('message', (data: unknown) => {
         expect(data).to.have.property('func', 'meow');
         done();
       });
@@ -206,7 +150,7 @@ describe('rpc', function () {
       it('stops all in-flight evaluations', async function () {
         messageBus = createMockRpcMesageBus();
         caller = createCaller(['neverResolves'], messageBus);
-        let err: Error;
+        let err!: Error;
         try {
           await Promise.all([
             caller.neverResolves(),
@@ -238,7 +182,7 @@ describe('rpc', function () {
         messageBus
       );
 
-      messageBus.on('message', (data: any) => {
+      messageBus.addEventListener('message', (data: any) => {
         // Due to how our mocks implemented we have to introduce an if here to
         // skip our own message being received by the message bus
         if (data.sender === 'postmsg-rpc/server') {
@@ -251,21 +195,11 @@ describe('rpc', function () {
         }
       });
 
-      messageBus.emit('message', {
+      messageBus.postMessage({
         sender: 'postmsg-rpc/client',
         func: 'meow',
         id: '123abc',
       });
     });
-
-    describe('close', function () {
-      it('disables all exposed listeners', function () {
-        messageBus = createMockRpcMesageBus();
-        exposed = exposeAll({ doSomething() {} }, messageBus);
-        expect(messageBus.listenerCount('message')).to.equal(1);
-        exposed[close]();
-        expect(messageBus.listenerCount('message')).to.equal(0);
-      });
-    });
   });
 });

package/src/rpc.ts

@@ -1,30 +1,12 @@
-import v8 from 'v8';
 import { expose, caller } from 'postmsg-rpc';
 import { deserializeError, serializeError } from './serializer';
-import type {
-  MessageData,
-  PostmsgRpcOptions,
-  ServerMessageData,
-  ClientMessageData,
-} from 'postmsg-rpc';
+import type { PostmsgRpcOptions } from 'postmsg-rpc';
 
-export function serialize(data: unknown): string {
-  return `data:;base64,${v8.serialize(data).toString('base64')}`;
-}
-
-export function deserialize<T = unknown>(str: string): T | string {
-  if (/^data:;base64,.+/.test(str)) {
-    return v8.deserialize(
-      Buffer.from(str.replace('data:;base64,', ''), 'base64')
-    );
-  }
-  return str;
-}
-
-type RPCMessageBus = { on: Function; off: Function } & (
-  | { postMessage: Function; send?: never }
-  | { postMessage?: never; send?: Function }
-);
+export type RPCMessageBus = {
+  postMessage: Function;
+  addEventListener: Function;
+  removeEventListener: Function;
+};
 
 enum RPCMessageTypes {
   Message,
@@ -47,81 +29,15 @@ function isRPCError(data: any): data is RPCError {
   );
 }
 
-function isMessageData(data: any): data is MessageData {
-  return data && typeof data === 'object' && 'id' in data && 'sender' in data;
-}
-
-function isServerMessageData(data: any): data is ServerMessageData {
-  return isMessageData(data) && data.sender === 'postmsg-rpc/server';
-}
-
-function isClientMessageData(data: any): data is ClientMessageData {
-  return isMessageData(data) && data.sender === 'postmsg-rpc/client';
-}
-
-export function removeTrailingUndefined(arr: unknown[]): unknown[] {
-  if (Array.isArray(arr)) {
-    arr = [...arr];
-    while (arr.length > 0 && arr[arr.length - 1] === undefined) {
-      arr.pop();
-    }
-  }
-  return arr;
-}
-
-function send(messageBus: RPCMessageBus, data: any): void {
-  if (
-    'postMessage' in messageBus &&
-    typeof messageBus.postMessage === 'function'
-  ) {
-    messageBus.postMessage(data);
-  }
-
-  if ('send' in messageBus && typeof messageBus.send === 'function') {
-    messageBus.send(data);
-  }
-}
-
 function getRPCOptions(messageBus: RPCMessageBus): PostmsgRpcOptions {
   return {
-    addListener: messageBus.on.bind(messageBus),
-    removeListener: messageBus.off.bind(messageBus),
+    addListener: messageBus.addEventListener.bind(messageBus),
+    removeListener: messageBus.removeEventListener.bind(messageBus),
     postMessage(data) {
-      if (isClientMessageData(data) && Array.isArray(data.args)) {
-        data.args = serialize(removeTrailingUndefined(data.args));
-      }
-
-      if (isServerMessageData(data)) {
-        // If serialization of the response failed for some reason (e.g., the
-        // value is not serializable) we want to propagate the error back to the
-        // client that issued the remote call instead of throwing on the server
-        // that was executing the method.
-        try {
-          data.res = serialize(data.res);
-        } catch (e: any) {
-          data.res = serialize({
-            type: RPCMessageTypes.Error,
-            payload: serializeError(e),
-          });
-        }
-      }
-
-      return send(messageBus, data);
+      return messageBus.postMessage(data);
     },
-    getMessageData(data) {
-      if (
-        isClientMessageData(data) &&
-        data.args &&
-        typeof data.args === 'string'
-      ) {
-        data.args = deserialize(data.args);
-      }
-
-      if (isServerMessageData(data) && typeof data.res === 'string') {
-        data.res = deserialize(data.res);
-      }
-
-      return data;
+    getMessageData(event) {
+      return (event as { data: unknown }).data ?? event;
     },
   };
 }
@@ -134,7 +50,10 @@ export type Exposed<T> = { [k in keyof T]: T[k] & { close(): void } } & {
   [close]: () => void;
 };
 
-export function exposeAll<O>(obj: O, messageBus: RPCMessageBus): Exposed<O> {
+export function exposeAll<O extends object>(
+  obj: O,
+  messageBus: RPCMessageBus
+): Exposed<O> {
   Object.entries(obj as Record<string, any>).forEach(([key, val]) => {
     const { close } = expose(
       key,
@@ -151,7 +70,7 @@ export function exposeAll<O>(obj: O, messageBus: RPCMessageBus): Exposed<O> {
       },
       getRPCOptions(messageBus)
     );
-    (val as any).close = close;
+    val.close = close;
   });
   Object.defineProperty(obj, close, {
     enumerable: false,
@@ -171,7 +90,7 @@ export type Caller<
   Keys extends keyof Impl = keyof Impl
 > = CancelableMethods<Pick<Impl, Keys>> & { [cancel]: () => void };
 
-export function createCaller<Impl extends {}>(
+export function createCaller<Impl extends object>(
   methodNames: Extract<keyof Impl, string>[],
   messageBus: RPCMessageBus,
   processors: Partial<