@effect/platform-node-shared 4.0.0-beta.1 → 4.0.0-beta.100

package/src/NodeChildProcessSpawner.ts

@@ -1,5 +1,19 @@
 /**
- * Node.js implementation of `ChildProcessSpawner`.
+ * Shared Node.js implementation of the child process spawner service.
+ *
+ * This module adapts `node:child_process.spawn` to the Effect
+ * `ChildProcessSpawner` service. Provide {@link layer} to run `ChildProcess`
+ * commands in Node-compatible runtimes: commands get scoped process handles
+ * with stdin sinks, stdout and stderr streams, exit-code waiting,
+ * interruption-time cleanup, process killing, and custom file-descriptor pipes.
+ *
+ * The implementation sits below the command-building API. It validates and
+ * resolves `cwd` through the Effect `FileSystem` and `Path` services,
+ * translates Node errno failures to `PlatformError`, and uses scopes to
+ * terminate referenced children when the owning effect is interrupted or
+ * finalized. Pipelines are flattened by {@link flattenCommand} and spawned one
+ * process at a time, wiring the selected source stream (`stdout`, `stderr`,
+ * `all`, or `fdN`) to the destination `stdin` or `fdN`.
  *
  * @since 4.0.0
  */
@@ -17,9 +31,15 @@ import * as Sink from "effect/Sink"
 import * as Stream from "effect/Stream"
 import * as ChildProcess from "effect/unstable/process/ChildProcess"
 import type { ChildProcessHandle } from "effect/unstable/process/ChildProcessSpawner"
-import { ChildProcessSpawner, ExitCode, makeHandle, ProcessId } from "effect/unstable/process/ChildProcessSpawner"
+import {
+  ChildProcessSpawner,
+  ExitCode,
+  make as makeSpawner,
+  makeHandle,
+  ProcessId
+} from "effect/unstable/process/ChildProcessSpawner"
 import * as NodeChildProcess from "node:child_process"
-import * as NodeJSStream from "node:stream"
+import { PassThrough } from "node:stream"
 import { handleErrnoException } from "./internal/utils.ts"
 import * as NodeSink from "./NodeSink.ts"
 import * as NodeStream from "./NodeStream.ts"
@@ -181,15 +201,15 @@ const make = Effect.gen(function*() {
         case "input": {
           // Create a sink to write to for input file descriptors
           let sink: Sink.Sink<void, Uint8Array, never, PlatformError.PlatformError> = Sink.drain
-          if (Predicate.isNotNullish(nodeStream) && "write" in nodeStream) {
+          if (nodeStream && "write" in nodeStream) {
             sink = NodeSink.fromWritable({
-              evaluate: () => nodeStream as NodeJS.WritableStream,
+              evaluate: () => nodeStream,
               onError: (error) => toPlatformError(`fromWritable(fd${fd})`, toError(error), command)
             })
           }
 
           // If user provided a stream, pipe it into the sink
-          if (Predicate.isNotUndefined(config.stream)) {
+          if (config.stream) {
             yield* Effect.forkScoped(Stream.run(config.stream, sink))
           }
 
@@ -200,15 +220,18 @@ const make = Effect.gen(function*() {
         case "output": {
           // Create a stream to read from for output file descriptors
           let stream: Stream.Stream<Uint8Array, PlatformError.PlatformError> = Stream.empty
-          if (Predicate.isNotNull(nodeStream) && Predicate.isNotUndefined(nodeStream) && "read" in nodeStream) {
+          if (nodeStream && "read" in nodeStream) {
+            const passThrough = new PassThrough()
+            nodeStream.on("error", (error) => passThrough.destroy(error))
+            nodeStream.pipe(passThrough)
             stream = NodeStream.fromReadable({
-              evaluate: () => nodeStream as NodeJS.ReadableStream,
+              evaluate: () => passThrough,
               onError: (error) => toPlatformError(`fromReadable(fd${fd})`, toError(error), command)
             })
           }
 
           // If user provided a sink, transduce the stream through it
-          if (Predicate.isNotUndefined(config.sink)) {
+          if (config.sink) {
             stream = Stream.transduce(stream, config.sink)
           }
 
@@ -251,35 +274,6 @@ const make = Effect.gen(function*() {
       return Effect.succeed(sink)
     })
 
-  /**
-   * Given that `NodeStream.fromReadable` uses `.read` to read data from the
-   * provided `Readable` stream, consumers would race to read data from the
-   * `handle.stdout` and `handle.stderr` streams if they were also simultaneously
-   * reading from the `handle.all` stream.
-   *
-   * To solve this, we leverage the fact that NodeJS `Readable` streams can be
-   * piped to multiple destinations simultaneously. The logic for the solution
-   * is as follows:
-   *
-   *   1. Pipe each original stream to two `PassThrough` streams:
-   *     - One dedicated PassThrough for individual access (.stdout / .stderr)
-   *     - One shared PassThrough for combined access (.all)
-   *   2. Create Effect streams from the PassThrough streams (not the originals)
-   *
-   * **Diagram**
-   *
-   *                                 ┌─────────────┐
-   *                           ┌────►│ passthrough │────► Effect stdout Stream
-   *                           │     └─────────────┘
-   *   childProcess.stdout ────┤
-   *                           │     ┌─────────────┐
-   *                           └────►│ passthrough │────► Effect all Stream
-   *                           ┌────►│             │
-   *   childProcess.stderr ────┤     └─────────────┘
-   *                           │     ┌─────────────┐
-   *                           └────►│ passthrough │────► Effect stderr Stream
-   *                                 └─────────────┘
-   */
   const setupChildOutputStreams = (
     command: ChildProcess.StandardCommand,
     childProcess: NodeChildProcess.ChildProcess,
@@ -290,85 +284,37 @@ const make = Effect.gen(function*() {
     stderr: Stream.Stream<Uint8Array, PlatformError.PlatformError>
     all: Stream.Stream<Uint8Array, PlatformError.PlatformError>
   } => {
-    const nodeStdout = childProcess.stdout
-    const nodeStderr = childProcess.stderr
-
-    // Create PassThrough streams for individual access to stdout and stderr.
-    // We pipe the original Node.js streams to these PassThroughs so that
-    // the data can be consumed by both the individual streams AND the
-    // combined stream (.all) simultaneously.
-    const stdoutPassThrough = Predicate.isNotNull(nodeStdout)
-      ? new NodeJSStream.PassThrough()
-      : null
-    const stderrPassThrough = Predicate.isNotNull(nodeStderr)
-      ? new NodeJSStream.PassThrough()
-      : null
-
-    // Create PassThrough for combined output (.all)
-    const combinedPassThrough = new NodeJSStream.PassThrough()
-
-    // Track stream endings for the combined stream
-    const totalStreams = (Predicate.isNotNull(nodeStdout) ? 1 : 0) +
-      (Predicate.isNotNull(nodeStderr) ? 1 : 0)
-
-    let endedCount = 0
-    const onStreamEnd = () => {
-      endedCount++
-      if (endedCount >= totalStreams) {
-        combinedPassThrough.end()
-      }
-    }
-
-    // Pipe stdout to both its own PassThrough and the combined PassThrough
-    if (Predicate.isNotNull(nodeStdout) && Predicate.isNotNull(stdoutPassThrough)) {
-      nodeStdout.pipe(stdoutPassThrough)
-      nodeStdout.pipe(combinedPassThrough, { end: false })
-      nodeStdout.once("end", onStreamEnd)
-    }
-
-    // Pipe stderr to both its own PassThrough and the combined PassThrough
-    if (Predicate.isNotNull(nodeStderr) && Predicate.isNotNull(stderrPassThrough)) {
-      nodeStderr.pipe(stderrPassThrough)
-      nodeStderr.pipe(combinedPassThrough, { end: false })
-      nodeStderr.once("end", onStreamEnd)
-    }
-
-    // Handle edge case: no streams available
-    if (totalStreams === 0) {
-      combinedPassThrough.end()
-    }
+    let stdout = childProcess.stdout ?
+      (() => {
+        const passThrough = new PassThrough()
+        childProcess.stdout!.on("error", (error) => passThrough.destroy(error))
+        childProcess.stdout!.pipe(passThrough)
+        return NodeStream.fromReadable({
+          evaluate: () => passThrough,
+          onError: (error) => toPlatformError("fromReadable(stdout)", toError(error), command)
+        })
+      })() :
+      Stream.empty
+    let stderr = childProcess.stderr ?
+      (() => {
+        const passThrough = new PassThrough()
+        childProcess.stderr!.on("error", (error) => passThrough.destroy(error))
+        childProcess.stderr!.pipe(passThrough)
+        return NodeStream.fromReadable({
+          evaluate: () => passThrough,
+          onError: (error) => toPlatformError("fromReadable(stderr)", toError(error), command)
+        })
+      })() :
+      Stream.empty
 
-    // Create Effect stream for stdout from its PassThrough
-    let stdout: Stream.Stream<Uint8Array, PlatformError.PlatformError> = Stream.empty
-    if (Predicate.isNotNull(stdoutPassThrough)) {
-      stdout = NodeStream.fromReadable({
-        evaluate: () => stdoutPassThrough,
-        onError: (error) => toPlatformError("fromReadable(stdout)", toError(error), command)
-      })
-      // Apply user-provided Sink if configured
-      if (Sink.isSink(stdoutConfig.stream)) {
-        stdout = Stream.transduce(stdout, stdoutConfig.stream)
-      }
+    if (Sink.isSink(stdoutConfig.stream)) {
+      stdout = Stream.transduce(stdout, stdoutConfig.stream)
     }
-
-    // Create Effect stream for stderr from its PassThrough
-    let stderr: Stream.Stream<Uint8Array, PlatformError.PlatformError> = Stream.empty
-    if (Predicate.isNotNull(stderrPassThrough)) {
-      stderr = NodeStream.fromReadable({
-        evaluate: () => stderrPassThrough,
-        onError: (error) => toPlatformError("fromReadable(stderr)", toError(error), command)
-      })
-      // Apply user-provided Sink if configured
-      if (Sink.isSink(stderrConfig.stream)) {
-        stderr = Stream.transduce(stderr, stderrConfig.stream)
-      }
+    if (Sink.isSink(stderrConfig.stream)) {
+      stderr = Stream.transduce(stderr, stderrConfig.stream)
     }
 
-    // Create Effect stream for combined output from the combined PassThrough
-    const all: Stream.Stream<Uint8Array, PlatformError.PlatformError> = NodeStream.fromReadable({
-      evaluate: () => combinedPassThrough,
-      onError: (error) => toPlatformError("fromReadable(all)", toError(error), command)
-    })
+    const all = Stream.merge(stdout, stderr)
 
     return { stdout, stderr, all }
   }
@@ -425,6 +371,23 @@ const make = Effect.gen(function*() {
     })
   }
 
+  const killProcessGroupOnExit = (
+    childProcess: NodeChildProcess.ChildProcess,
+    signal: NodeJS.Signals
+  ): void => {
+    if (globalThis.process.platform === "win32") {
+      NodeChildProcess.exec(`taskkill /pid ${childProcess.pid} /T /F`, () => {
+        // ignore errors during best-effort cleanup
+      })
+      return
+    }
+    try {
+      globalThis.process.kill(-childProcess.pid!, signal)
+    } catch {
+      // ignore errors during best-effort cleanup
+    }
+  }
+
   const killProcess = (
     command: ChildProcess.StandardCommand,
     childProcess: NodeChildProcess.ChildProcess,
@@ -456,7 +419,7 @@ const make = Effect.gen(function*() {
       ? kill(command, childProcess, killSignal)
       : Effect.timeoutOrElse(kill(command, childProcess, killSignal), {
         duration: options.forceKillAfter,
-        onTimeout: () => kill(command, childProcess, "SIGKILL")
+        orElse: () => kill(command, childProcess, "SIGKILL")
       })
   }
 
@@ -501,6 +464,7 @@ const make = Effect.gen(function*() {
         const stdoutConfig = resolveOutputOption(cmd.options, "stdout")
         const stderrConfig = resolveOutputOption(cmd.options, "stderr")
         const resolvedAdditionalFds = resolveAdditionalFds(cmd.options)
+        let isReferenced = true
 
         const cwd = yield* resolveWorkingDirectory(cmd.options)
         const env = resolveEnvironment(cmd.options)
@@ -526,20 +490,40 @@ const make = Effect.gen(function*() {
               }
               return yield* Effect.void
             }
+            if (!isReferenced) {
+              return yield* Effect.void
+            }
             // Process is still running, kill it
             return yield* killWithTimeout((command, childProcess, signal) =>
-              Effect.catch(
-                killProcessGroup(command, childProcess, signal),
-                () => killProcess(command, childProcess, signal)
+              killProcessGroup(command, childProcess, signal).pipe(
+                Effect.catch(() => killProcess(command, childProcess, signal)),
+                Effect.andThen(Deferred.await(exitSignal))
               )
             ).pipe(
-              Effect.andThen(Deferred.await(exitSignal)),
               Effect.ignore
             )
           })
         )
 
         const pid = ProcessId(childProcess.pid!)
+        childProcess.on("exit", (code) => {
+          if (code !== 0 && Predicate.isNotNull(code)) {
+            killProcessGroupOnExit(childProcess, cmd.options.killSignal ?? "SIGTERM")
+          }
+        })
+        const reref = Effect.sync(() => {
+          if (!isReferenced) {
+            childProcess.ref()
+            isReferenced = true
+          }
+        })
+        const unref = Effect.sync(() => {
+          if (isReferenced) {
+            childProcess.unref()
+            isReferenced = false
+          }
+          return reref
+        })
         const stdin = yield* setupChildStdin(cmd, childProcess, stdinConfig)
         const { all, stderr, stdout } = setupChildOutputStreams(cmd, childProcess, stdoutConfig, stderrConfig)
         const { getInputFd, getOutputFd } = yield* setupAdditionalFds(cmd, childProcess, resolvedAdditionalFds)
@@ -557,12 +541,11 @@ const make = Effect.gen(function*() {
         const kill = (options?: ChildProcess.KillOptions | undefined) => {
           const killWithTimeout = withTimeout(childProcess, cmd, options)
           return killWithTimeout((command, childProcess, signal) =>
-            Effect.catch(
-              killProcessGroup(command, childProcess, signal),
-              () => killProcess(command, childProcess, signal)
+            killProcessGroup(command, childProcess, signal).pipe(
+              Effect.catch(() => killProcess(command, childProcess, signal)),
+              Effect.andThen(Deferred.await(exitSignal))
             )
           ).pipe(
-            Effect.andThen(Deferred.await(exitSignal)),
             Effect.asVoid
           )
         }
@@ -577,14 +560,15 @@ const make = Effect.gen(function*() {
           stderr,
           all,
           getInputFd,
-          getOutputFd
+          getOutputFd,
+          unref
         })
       }
       case "PipedCommand": {
         const { commands, pipeOptions } = flattenCommand(cmd)
         const [root, ...pipeline] = commands
 
-        let handle = spawnCommand(root)
+        const handles = [yield* spawnCommand(root)]
 
         for (let i = 0; i < pipeline.length; i++) {
           const command = pipeline[i]
@@ -593,7 +577,7 @@ const make = Effect.gen(function*() {
 
           // Get the appropriate stream from the source based on `from` option
           const sourceStream = Stream.unwrap(
-            Effect.map(handle, (h) => getSourceStream(h, options.from))
+            Effect.succeed(getSourceStream(handles[handles.length - 1], options.from))
           )
 
           // Determine where to pipe: stdin or custom fd
@@ -601,48 +585,73 @@ const make = Effect.gen(function*() {
 
           if (toOption === "stdin") {
             // Pipe to stdin (default behavior)
-            handle = spawnCommand(ChildProcess.make(command.command, command.args, {
-              ...command.options,
-              stdin: { ...stdinConfig, stream: sourceStream }
-            }))
+            handles.push(
+              yield* spawnCommand(ChildProcess.make(command.command, command.args, {
+                ...command.options,
+                stdin: { ...stdinConfig, stream: sourceStream }
+              }))
+            )
           } else {
             // Pipe to custom fd (fd3, fd4, etc.)
             const fd = ChildProcess.parseFdName(toOption)
             if (Predicate.isNotUndefined(fd)) {
               const fdName = ChildProcess.fdName(fd) as `fd${number}`
               const existingFds = command.options.additionalFds ?? {}
-              handle = spawnCommand(ChildProcess.make(command.command, command.args, {
-                ...command.options,
-                additionalFds: {
-                  ...existingFds,
-                  [fdName]: { type: "input" as const, stream: sourceStream }
-                }
-              }))
+              handles.push(
+                yield* spawnCommand(ChildProcess.make(command.command, command.args, {
+                  ...command.options,
+                  additionalFds: {
+                    ...existingFds,
+                    [fdName]: { type: "input" as const, stream: sourceStream }
+                  }
+                }))
+              )
             } else {
               // Invalid fd name, fall back to stdin
-              handle = spawnCommand(ChildProcess.make(command.command, command.args, {
-                ...command.options,
-                stdin: { ...stdinConfig, stream: sourceStream }
-              }))
+              handles.push(
+                yield* spawnCommand(ChildProcess.make(command.command, command.args, {
+                  ...command.options,
+                  stdin: { ...stdinConfig, stream: sourceStream }
+                }))
+              )
             }
           }
         }
 
-        return yield* handle
+        const handle = handles[handles.length - 1]
+        const unref = Effect.gen(function*() {
+          const rerefs: Array<Effect.Effect<void, PlatformError.PlatformError>> = []
+          for (const handle of handles) {
+            rerefs.push(yield* handle.unref)
+          }
+          return Effect.forEach([...rerefs].reverse(), (reref) => reref, { discard: true })
+        })
+
+        return makeHandle({
+          pid: handle.pid,
+          exitCode: handle.exitCode,
+          isRunning: handle.isRunning,
+          kill: handle.kill,
+          stdin: handle.stdin,
+          stdout: handle.stdout,
+          stderr: handle.stderr,
+          all: handle.all,
+          getInputFd: handle.getInputFd,
+          getOutputFd: handle.getOutputFd,
+          unref
+        })
       }
     }
   })
 
-  return ChildProcessSpawner.of({
-    spawn: spawnCommand
-  })
+  return makeSpawner(spawnCommand)
 })
 
 /**
- * Layer providing the `NodeChildProcessSpawner` implementation.
+ * Layer that provides the `NodeChildProcessSpawner` implementation.
  *
+ * @category layers
  * @since 4.0.0
- * @category Layers
  */
 export const layer: Layer.Layer<
   ChildProcessSpawner,
@@ -657,8 +666,8 @@ export const layer: Layer.Layer<
 /**
  * Result of flattening a pipeline of commands.
  *
+ * @category models
  * @since 4.0.0
- * @category Models
  */
 export interface FlattenedPipeline {
   readonly commands: Arr.NonEmptyReadonlyArray<ChildProcess.StandardCommand>
@@ -669,8 +678,8 @@ export interface FlattenedPipeline {
  * Flattens a `Command` into an array of `StandardCommand`s along with pipe
  * options for each connection.
  *
+ * @category transforming
  * @since 4.0.0
- * @category Utilities
  */
 export const flattenCommand = (
   command: ChildProcess.Command

package/src/NodeClusterSocket.ts

@@ -1,8 +1,16 @@
 /**
- * @since 1.0.0
+ * Node TCP socket transport for Effect Cluster runner-to-runner RPC.
+ *
+ * This module provides the shared Node layers used by socket-based cluster
+ * transports. `layerClientProtocol` opens TCP sockets to peer runner addresses
+ * and wraps them in the current RPC serialization protocol. `layerSocketServer`
+ * exposes the socket server that receives incoming runner RPC traffic.
+ *
+ * @since 4.0.0
  */
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
 import * as Runners from "effect/unstable/cluster/Runners"
 import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig"
 import * as RpcClient from "effect/unstable/rpc/RpcClient"
@@ -13,8 +21,11 @@ import * as NodeSocket from "./NodeSocket.ts"
 import * as NodeSocketServer from "./NodeSocketServer.ts"
 
 /**
- * @since 1.0.0
- * @category Layers
+ * Provides the cluster `RpcClientProtocol` by opening TCP sockets to runner
+ * addresses and using the current RPC serialization service.
+ *
+ * @category layers
+ * @since 4.0.0
  */
 export const layerClientProtocol: Layer.Layer<
   Runners.RpcClientProtocol,
@@ -39,8 +50,11 @@ export const layerClientProtocol: Layer.Layer<
 )
 
 /**
- * @since 1.0.0
- * @category Layers
+ * Provides the socket server used by cluster runners, listening on
+ * `ShardingConfig.runnerListenAddress` or `runnerAddress`.
+ *
+ * @category layers
+ * @since 4.0.0
  */
 export const layerSocketServer: Layer.Layer<
   SocketServer.SocketServer,
@@ -48,9 +62,9 @@ export const layerSocketServer: Layer.Layer<
   ShardingConfig.ShardingConfig
 > = Effect.gen(function*() {
   const config = yield* ShardingConfig.ShardingConfig
-  const listenAddress = config.runnerListenAddress ?? config.runnerAddress
-  if (listenAddress === undefined) {
+  const listenAddress = Option.orElse(config.runnerListenAddress, () => config.runnerAddress)
+  if (Option.isNone(listenAddress)) {
     return yield* Effect.die("layerSocketServer: ShardingConfig.runnerListenAddress is None")
   }
-  return NodeSocketServer.layer(listenAddress)
+  return NodeSocketServer.layer(listenAddress.value)
 }).pipe(Layer.unwrap)

package/src/NodeCrypto.ts

@@ -0,0 +1,60 @@
+/**
+ * Node-compatible implementation of Effect's `Crypto` service.
+ *
+ * This module builds the service from `node:crypto`, using `randomBytes` for
+ * random data and `createHash` for supported digest algorithms. It exports
+ * `make` as the concrete service value and `layer` for providing it through
+ * Effect context.
+ *
+ * @since 1.0.0
+ */
+import * as EffectCrypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as PlatformError from "effect/PlatformError"
+import * as NodeCrypto from "node:crypto"
+
+const toHashAlgorithm = (algorithm: EffectCrypto.DigestAlgorithm): string => {
+  switch (algorithm) {
+    case "SHA-1":
+      return "sha1"
+    case "SHA-256":
+      return "sha256"
+    case "SHA-384":
+      return "sha384"
+    case "SHA-512":
+      return "sha512"
+  }
+}
+
+const digest: EffectCrypto.Crypto["digest"] = (algorithm, data) =>
+  Effect.try({
+    try: () => Uint8Array.from(NodeCrypto.createHash(toHashAlgorithm(algorithm)).update(data).digest()),
+    catch: (cause) =>
+      PlatformError.systemError({
+        module: "Crypto",
+        method: "digest",
+        _tag: "Unknown",
+        description: "Could not compute digest",
+        cause
+      })
+  })
+
+/**
+ * The default Node.js Crypto service implementation.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const make: EffectCrypto.Crypto = EffectCrypto.make({
+  randomBytes: NodeCrypto.randomBytes,
+  digest
+})
+
+/**
+ * Layer that provides the Node.js Crypto service implementation.
+ *
+ * @category layers
+ * @since 1.0.0
+ */
+export const layer: Layer.Layer<EffectCrypto.Crypto> = Layer.succeed(EffectCrypto.Crypto, make)