@parallel-park/in-child-process 0.3.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License Copyright (c) 2022-2026 Lily Skye
+
+Permission is hereby granted, free of
+charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to the
+following conditions:
+
+The above copyright notice and this permission notice
+(including the next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,129 @@
+# @parallel-park/in-child-process
+
+Run code within a child process.
+
+## Usage
+
+`@parallel-park/in-child-process` exports one function: `inChildProcess`.
+
+### `inChildProcess`
+
+`inChildProcess` is a function that you pass a function into, and it spawns a separate node process to run your function in. Once your function has completed, its return/resolved value will be sent back to the node process you called `inChildProcess` from.
+
+```ts
+import { inChildProcess } from "parallel-park";
+
+const result = await inChildProcess(() => {
+  return 2 + 2;
+});
+
+console.log(result); // 4
+```
+
+Your function can also return a Promise:
+
+```ts
+// Either by returning a Promise directly...
+await inChildProcess(() => {
+  return Promise.resolve(2 + 2);
+});
+
+// Or by using an async function, which returns a Promise that resolves to the function's return value
+await inChildProcess(async () => {
+  return 2 + 2;
+});
+```
+
+> ⚠️ NOTE: The return value of your function must be JSON-serializable, or else it won't make it across the gap between the parent node process and the child one.
+
+The function you pass into `inChildProcess` will be executed in a separate node process; as such, it won't be able to access variables defined in the file calling `inChildProcess`:
+
+```ts
+const myName = "Lily";
+
+await inChildProcess(() => {
+  // Throws an error: myName is not defined
+  return myName + "!";
+});
+```
+
+To work around this, you can pass an object into `inChildProcess` as its first argument, before the function. When called this way, the function will receive that object:
+
+```ts
+await inChildProcess({ myName: "Lily" }, (data) => {
+  const myName = data.myName;
+
+  // No longer throws an error
+  return myName + "!";
+});
+```
+
+It's common to use object shorthand and destructuring syntax when passing values in:
+
+```ts
+const myName = "Lily";
+
+await inChildProcess({ myName }, ({ myName }) => {
+  return myName + "!";
+});
+```
+
+> ⚠️ NOTE: The values in your input object must be JSON-serializable, or else they won't make it across the gap between the parent node process and the child one.
+
+Because the inputs have to be JSON-serializable, you may run into an issue if trying to use an external module within the child process:
+
+```ts
+const util = require("util");
+
+await inChildProcess({ util }, ({ util }) => {
+  const someData = { something: true };
+  // Throws an error: util.inspect is not a function
+  return util.inspect(someData);
+});
+```
+
+To work around this, call `require` inside the child process function:
+
+```ts
+await inChildProcess(() => {
+  const util = require("util"); // the require is inside the function now
+
+  const someData = { something: true };
+
+  // No longer throws an error
+  return util.inspect(someData);
+});
+```
+
+If you want to use the external module both inside of the child process and outside of it, `require` it in both places:
+
+```ts
+const util = require("util");
+
+await inChildProcess(() => {
+  const util = require("util");
+
+  const someData = { something: true };
+
+  // No longer throws an error
+  return util.inspect(someData);
+});
+```
+
+The `require` inside of the child process can also be used to load stuff from your own code into the child process:
+
+```ts
+const file = "/home/lily/hello.txt";
+
+await inChildProcess({ file }, async ({ file }) => {
+  const processFile = require("./process-file");
+
+  const results = await processFile(file);
+  console.log(results);
+  return results;
+});
+```
+
+## License
+
+MIT

package/dist/child-process-worker.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const vm_1 = __importDefault(require("vm"));
+const make_module_env_1 = __importDefault(require("make-module-env"));
+const debug_1 = __importDefault(require("debug"));
+const read_until_end_1 = require("./read-until-end");
+const path_1 = __importDefault(require("path"));
+const debug = (0, debug_1.default)("parallel-park:child-process-worker");
+const commsIn = fs_1.default.createReadStream(
+// @ts-ignore
+null, { fd: 3 });
+const commsOut = fs_1.default.createWriteStream(
+// @ts-ignore
+null, { fd: 4 });
+debug("reading input data...");
+(0, read_until_end_1.readUntilEnd)(commsIn)
+    .then((data) => {
+    debug("parsing input data...");
+    try {
+        const [inputs, fnString, callingFile] = JSON.parse(data);
+        onReady(inputs, fnString, callingFile);
+    }
+    catch (err) {
+        onError(err);
+    }
+})
+    .catch(onError);
+function onReady(inputs, fnString, callingFile) {
+    debug("in onReady %o", { inputs, fnString, callingFile });
+    // Relevant when callingFile is eg. "REPL2" (from Node.js repl)
+    if (!path_1.default.isAbsolute(callingFile)) {
+        callingFile = path_1.default.join(process.cwd(), "fake-path.js");
+    }
+    const wrapperFn = vm_1.default.runInThisContext(`(function moduleWrapper(exports, require, module, __filename, __dirname) {
+  return ${fnString};})`);
+    const env = (0, make_module_env_1.default)(callingFile);
+    const fn = wrapperFn(env.exports, env.require, env.module, env.__filename, env.__dirname);
+    const result = fn(inputs);
+    if (typeof result === "object" &&
+        result != null &&
+        typeof result.then === "function") {
+        result.then(onSuccess, onError);
+    }
+    else {
+        onSuccess(result);
+    }
+}
+function onSuccess(data) {
+    debug("in onSuccess %o", { data });
+    commsOut.end(JSON.stringify({ type: "success", data }));
+}
+function onError(error) {
+    debug("in onError %o", { error });
+    commsOut.end(JSON.stringify({
+        type: "error",
+        error: {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+        },
+    }));
+}

package/dist/in-child-process.js

@@ -0,0 +1,134 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inChildProcess = void 0;
+const child_process_1 = __importDefault(require("child_process"));
+const error_utils_1 = require("@suchipi/error-utils");
+const debug_1 = __importDefault(require("debug"));
+const util_1 = __importDefault(require("util"));
+const read_until_end_1 = require("./read-until-end");
+const debug = (0, debug_1.default)("parallel-park:in-child-process");
+const runnerPath = require.resolve("../dist/child-process-worker");
+const inChildProcess = (...args) => {
+    var _a;
+    const inputs = typeof args[0] === "function" ? {} : args[0];
+    const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
+    if (typeof inputs !== "object") {
+        throw new Error("The first argument to inChildProcess should be an object of input data to pass to the child process.");
+    }
+    if (typeof functionToRun !== "function") {
+        throw new Error("The second argument to inChildProcess should be a function to run in the child process.");
+    }
+    const here = new error_utils_1.ParsedError(new Error("here"));
+    const callingFrame = here.stackFrames[1];
+    const callingFile = (_a = callingFrame === null || callingFrame === void 0 ? void 0 : callingFrame.fileName) !== null && _a !== void 0 ? _a : "unknown file";
+    debug("spawning child process: %o", [process.argv[0], runnerPath]);
+    const child = child_process_1.default.spawn(process.argv[0], [runnerPath], {
+        stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
+    });
+    return new Promise((resolve, reject) => {
+        child.on("error", reject);
+        const commsOut = child.stdio[3];
+        const commsIn = child.stdio[4];
+        child.on("spawn", () => {
+            const dataToSend = JSON.stringify([
+                inputs,
+                functionToRun.toString(),
+                callingFile,
+            ]);
+            debug("sending inputs to child process: %o", dataToSend);
+            commsOut.end(dataToSend, "utf-8");
+        });
+        let receivedData = "";
+        (0, read_until_end_1.readUntilEnd)(commsIn).then((data) => {
+            debug("received data from child process: %o", data);
+            receivedData = data;
+        });
+        child.on("close", (code, signal) => {
+            debug("child process closed: %o", { code, signal });
+            if (code !== 0) {
+                reject(new Error(`Child process exited with nonzero status code: ${JSON.stringify({
+                    code,
+                    signal,
+                })}`));
+            }
+            else {
+                debug("parsing received data from child process...");
+                let result;
+                try {
+                    result = JSON.parse(receivedData);
+                }
+                catch (err) {
+                    reject(new Error(`parallel-park error: failed to parse received data as JSON. data was: ${util_1.default.inspect(receivedData, { colors: true, depth: Infinity })}`));
+                    return;
+                }
+                switch (result.type) {
+                    case "success": {
+                        debug("child process finished successfully with result: %o", result.data);
+                        resolve(result.data);
+                        break;
+                    }
+                    case "error": {
+                        debug("child process errored: %o", result.error);
+                        const error = new Error(result.error.message);
+                        Object.defineProperty(error, "name", { value: result.error.name });
+                        Object.defineProperty(error, "stack", {
+                            value: result.error.name +
+                                ": " +
+                                result.error.message +
+                                "\n" +
+                                result.error.stack
+                                    .split("\n")
+                                    .slice(1)
+                                    .filter((line) => !/node:internal|node:events/.test(line))
+                                    .map((line) => {
+                                    if (/evalmachine/.test(line)) {
+                                        const lineWithoutEvalMachine = line.replace(/evalmachine(?:\.<anonymous>)?/, "<function passed into inChildProcess>");
+                                        const matches = line.match(/:(\d+):(\d+)\)?$/);
+                                        if (!matches) {
+                                            return lineWithoutEvalMachine;
+                                        }
+                                        else {
+                                            let [_, row, col] = matches;
+                                            // subtract 1 from row to skip the module wrapper function line
+                                            row = row - 1;
+                                            // subtract the length of the `return ` keywords in front of the function
+                                            if (row === 1) {
+                                                col = col - `return `.length;
+                                            }
+                                            const hadParen = /\)$/.test(lineWithoutEvalMachine);
+                                            return lineWithoutEvalMachine.replace(/:\d+:\d+\)?$/, `:${row}:${col - 1}${hadParen ? ")" : ""}`);
+                                        }
+                                    }
+                                    else {
+                                        return line;
+                                    }
+                                })
+                                    .join("\n") +
+                                "\n" +
+                                error
+                                    .stack.split("\n")
+                                    .slice(1)
+                                    .filter((line) => !/node:internal|node:events/.test(line))
+                                    .join("\n") +
+                                "\n" +
+                                here
+                                    .stack.split("\n")
+                                    .slice(2)
+                                    .filter((line) => !/node:internal|node:events/.test(line))
+                                    .join("\n"),
+                        });
+                        reject(error);
+                        break;
+                    }
+                    default: {
+                        reject(new Error(`Internal parallel-park error: unhandled result type: ${result.type}`));
+                    }
+                }
+            }
+        });
+    });
+};
+exports.inChildProcess = inChildProcess;

package/dist/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inChildProcess = void 0;
+var in_child_process_1 = require("./in-child-process");
+Object.defineProperty(exports, "inChildProcess", { enumerable: true, get: function () { return in_child_process_1.inChildProcess; } });

package/dist/read-until-end.js

@@ -0,0 +1,25 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readUntilEnd = readUntilEnd;
+const debug_1 = __importDefault(require("debug"));
+const debug = (0, debug_1.default)("parallel-park:read-until-end");
+let streamId = 0;
+function readUntilEnd(stream) {
+    const id = `${process.pid}-${streamId}`;
+    streamId++;
+    return new Promise((resolve) => {
+        let data = "";
+        stream.on("data", (chunk) => {
+            const chunkStr = chunk.toString("utf-8");
+            debug("received data chunk from stream %s: %o", id, chunkStr);
+            data += chunkStr;
+        });
+        stream.on("close", () => {
+            debug("stream %s closed; resolving with: %o", id, data);
+            resolve(data);
+        });
+    });
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@parallel-park/in-child-process",
+  "version": "0.3.0",
+  "description": "Run code within a child process",
+  "main": "dist/index.js",
+  "types": "src/index.ts",
+  "scripts": {
+    "test": "vitest run",
+    "build": "rm -rf ./dist && tsc"
+  },
+  "keywords": [
+    "parallel",
+    "concurrent",
+    "thread",
+    "multithread",
+    "worker",
+    "process",
+    "child",
+    "concurrency",
+    "co-operative",
+    "promise",
+    "map"
+  ],
+  "author": "Lily Skye <me@suchipi.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/suchipi/parallel-park.git"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.12"
+  },
+  "dependencies": {
+    "@suchipi/error-utils": "^0.2.1",
+    "debug": "^4.4.3",
+    "make-module-env": "^1.1.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/child-process-worker.ts

@@ -0,0 +1,86 @@
+import fs from "fs";
+import vm from "vm";
+import makeModuleEnv from "make-module-env";
+import makeDebug from "debug";
+import { readUntilEnd } from "./read-until-end";
+import path from "path";
+
+const debug = makeDebug("parallel-park:child-process-worker");
+
+const commsIn = fs.createReadStream(
+  // @ts-ignore
+  null,
+  { fd: 3 }
+);
+const commsOut = fs.createWriteStream(
+  // @ts-ignore
+  null,
+  { fd: 4 }
+);
+
+debug("reading input data...");
+readUntilEnd(commsIn)
+  .then((data) => {
+    debug("parsing input data...");
+    try {
+      const [inputs, fnString, callingFile] = JSON.parse(data);
+      onReady(inputs, fnString, callingFile);
+    } catch (err) {
+      onError(err as Error);
+    }
+  })
+  .catch(onError);
+
+function onReady(inputs: any, fnString: string, callingFile: string) {
+  debug("in onReady %o", { inputs, fnString, callingFile });
+
+  // Relevant when callingFile is eg. "REPL2" (from Node.js repl)
+  if (!path.isAbsolute(callingFile)) {
+    callingFile = path.join(process.cwd(), "fake-path.js");
+  }
+
+  const wrapperFn = vm.runInThisContext(
+    `(function moduleWrapper(exports, require, module, __filename, __dirname) {
+  return ${fnString};})`
+  );
+  const env = makeModuleEnv(callingFile);
+  const fn = wrapperFn(
+    env.exports,
+    env.require,
+    env.module,
+    env.__filename,
+    env.__dirname
+  );
+  const result = fn(inputs);
+
+  if (
+    typeof result === "object" &&
+    result != null &&
+    typeof result.then === "function"
+  ) {
+    result.then(onSuccess, onError);
+  } else {
+    onSuccess(result);
+  }
+}
+
+function onSuccess(data: any) {
+  debug("in onSuccess %o", { data });
+
+  commsOut.end(JSON.stringify({ type: "success", data }));
+}
+
+function onError(error: Error) {
+  debug("in onError %o", { error });
+
+  commsOut.end(
+    JSON.stringify({
+      type: "error",
+      error: {
+        name: error.name,
+        message: error.message,
+        stack: error.stack,
+      },
+    })
+  );
+}

package/src/in-child-process.ts

@@ -0,0 +1,180 @@
+import child_process from "child_process";
+import type * as stream from "stream";
+import { ParsedError } from "@suchipi/error-utils";
+import makeDebug from "debug";
+import util from "util";
+import { readUntilEnd } from "./read-until-end";
+
+const debug = makeDebug("parallel-park:in-child-process");
+
+const runnerPath = require.resolve("../dist/child-process-worker");
+
+type InChildProcess = {
+  <Inputs extends { [key: string]: any }, Result>(
+    inputs: Inputs,
+    functionToRun: (inputs: Inputs) => Result | Promise<Result>
+  ): Promise<Result>;
+
+  <Result>(functionToRun: () => Result | Promise<Result>): Promise<Result>;
+};
+
+export const inChildProcess: InChildProcess = (...args: Array<any>) => {
+  const inputs = typeof args[0] === "function" ? {} : args[0];
+  const functionToRun = typeof args[0] === "function" ? args[0] : args[1];
+
+  if (typeof inputs !== "object") {
+    throw new Error(
+      "The first argument to inChildProcess should be an object of input data to pass to the child process."
+    );
+  }
+
+  if (typeof functionToRun !== "function") {
+    throw new Error(
+      "The second argument to inChildProcess should be a function to run in the child process."
+    );
+  }
+
+  const here = new ParsedError(new Error("here"));
+
+  const callingFrame = here.stackFrames[1];
+  const callingFile = callingFrame?.fileName ?? "unknown file";
+
+  debug("spawning child process: %o", [process.argv[0], runnerPath]);
+
+  const child = child_process.spawn(process.argv[0], [runnerPath], {
+    stdio: ["inherit", "inherit", "inherit", "pipe", "pipe"],
+  });
+
+  return new Promise((resolve, reject) => {
+    child.on("error", reject);
+
+    const commsOut: stream.Writable = child.stdio![3] as any;
+    const commsIn: stream.Readable = child.stdio![4] as any;
+
+    child.on("spawn", () => {
+      const dataToSend = JSON.stringify([
+        inputs,
+        functionToRun.toString(),
+        callingFile,
+      ]);
+      debug("sending inputs to child process: %o", dataToSend);
+      commsOut.end(dataToSend, "utf-8");
+    });
+
+    let receivedData = "";
+    readUntilEnd(commsIn).then((data) => {
+      debug("received data from child process: %o", data);
+      receivedData = data;
+    });
+
+    child.on("close", (code, signal) => {
+      debug("child process closed: %o", { code, signal });
+
+      if (code !== 0) {
+        reject(
+          new Error(
+            `Child process exited with nonzero status code: ${JSON.stringify({
+              code,
+              signal,
+            })}`
+          )
+        );
+      } else {
+        debug("parsing received data from child process...");
+        let result: any;
+        try {
+          result = JSON.parse(receivedData);
+        } catch (err) {
+          reject(
+            new Error(
+              `parallel-park error: failed to parse received data as JSON. data was: ${util.inspect(
+                receivedData,
+                { colors: true, depth: Infinity }
+              )}`
+            )
+          );
+          return;
+        }
+        switch (result.type) {
+          case "success": {
+            debug(
+              "child process finished successfully with result: %o",
+              result.data
+            );
+            resolve(result.data);
+            break;
+          }
+          case "error": {
+            debug("child process errored: %o", result.error);
+            const error = new Error(result.error.message);
+            Object.defineProperty(error, "name", { value: result.error.name });
+            Object.defineProperty(error, "stack", {
+              value:
+                result.error.name +
+                ": " +
+                result.error.message +
+                "\n" +
+                result.error.stack
+                  .split("\n")
+                  .slice(1)
+                  .filter((line) => !/node:internal|node:events/.test(line))
+                  .map((line) => {
+                    if (/evalmachine/.test(line)) {
+                      const lineWithoutEvalMachine = line.replace(
+                        /evalmachine(?:\.<anonymous>)?/,
+                        "<function passed into inChildProcess>"
+                      );
+
+                      const matches = line.match(/:(\d+):(\d+)\)?$/);
+                      if (!matches) {
+                        return lineWithoutEvalMachine;
+                      } else {
+                        let [_, row, col] = matches;
+                        // subtract 1 from row to skip the module wrapper function line
+                        row = row - 1;
+
+                        // subtract the length of the `return ` keywords in front of the function
+                        if (row === 1) {
+                          col = col - `return `.length;
+                        }
+
+                        const hadParen = /\)$/.test(lineWithoutEvalMachine);
+
+                        return lineWithoutEvalMachine.replace(
+                          /:\d+:\d+\)?$/,
+                          `:${row}:${col - 1}${hadParen ? ")" : ""}`
+                        );
+                      }
+                    } else {
+                      return line;
+                    }
+                  })
+                  .join("\n") +
+                "\n" +
+                error
+                  .stack!.split("\n")
+                  .slice(1)
+                  .filter((line) => !/node:internal|node:events/.test(line))
+                  .join("\n") +
+                "\n" +
+                here
+                  .stack!.split("\n")
+                  .slice(2)
+                  .filter((line) => !/node:internal|node:events/.test(line))
+                  .join("\n"),
+            });
+            reject(error);
+            break;
+          }
+          default: {
+            reject(
+              new Error(
+                `Internal parallel-park error: unhandled result type: ${result.type}`
+              )
+            );
+          }
+        }
+      }
+    });
+  });
+};

package/src/index.ts

@@ -0,0 +1 @@
+export { inChildProcess } from "./in-child-process";

package/src/read-until-end.ts

@@ -0,0 +1,25 @@
+import type stream from "stream";
+import makeDebug from "debug";
+const debug = makeDebug("parallel-park:read-until-end");
+
+let streamId = 0;
+
+export function readUntilEnd(stream: stream.Readable): Promise<string> {
+  const id = `${process.pid}-${streamId}`;
+  streamId++;
+
+  return new Promise((resolve) => {
+    let data = "";
+
+    stream.on("data", (chunk) => {
+      const chunkStr = chunk.toString("utf-8");
+      debug("received data chunk from stream %s: %o", id, chunkStr);
+      data += chunkStr;
+    });
+
+    stream.on("close", () => {
+      debug("stream %s closed; resolving with: %o", id, data);
+      resolve(data);
+    });
+  });
+}

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+
+  "include": ["./src/**/*.ts"],
+  "exclude": ["./**/*.test.ts"],
+
+  "compilerOptions": {
+    "outDir": "./dist"
+  }
+}