process_executer 4.0.4 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.editorconfig +10 -0
- data/.release-please-manifest.json +1 -1
- data/.rubocop.yml +13 -0
- data/.vscode/extensions.json +3 -0
- data/CHANGELOG.md +42 -0
- data/README.md +20 -0
- data/Rakefile +3 -0
- data/lib/process_executer/commands/run.rb +152 -10
- data/lib/process_executer/commands/run_with_capture.rb +41 -4
- data/lib/process_executer/commands/spawn_with_timeout.rb +185 -16
- data/lib/process_executer/destinations/monitored_pipe.rb +7 -4
- data/lib/process_executer/errors.rb +2 -2
- data/lib/process_executer/monitored_pipe.rb +373 -44
- data/lib/process_executer/options/base.rb +32 -3
- data/lib/process_executer/options/run_with_capture_options.rb +70 -8
- data/lib/process_executer/options/spawn_options.rb +13 -0
- data/lib/process_executer/result.rb +13 -2
- data/lib/process_executer/version.rb +1 -1
- data/lib/process_executer.rb +34 -2
- data/process_spawn_test/README.md +38 -1
- data/process_spawn_test/spec/spec_helper.rb +18 -0
- data/process_spawn_test/spec/test_spec.rb +21 -0
- data/release-please-config.json +1 -0
- metadata +6 -4
|
@@ -25,12 +25,15 @@ module ProcessExecuter
|
|
|
25
25
|
destination.write data
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
#
|
|
28
|
+
# Does nothing: closing a caller-provided pipe is the caller's responsibility
|
|
29
|
+
#
|
|
30
|
+
# The library closes only pipes it creates. A {ProcessExecuter::MonitoredPipe}
|
|
31
|
+
# given as a redirection destination was created by the caller, and the
|
|
32
|
+
# {ProcessExecuter::MonitoredPipe} documentation makes closing it the
|
|
33
|
+
# caller's responsibility.
|
|
29
34
|
#
|
|
30
35
|
# @return [void]
|
|
31
|
-
def close
|
|
32
|
-
destination.close if destination.state == :open
|
|
33
|
-
end
|
|
36
|
+
def close; end
|
|
34
37
|
|
|
35
38
|
# Determines if this class can handle the given destination
|
|
36
39
|
#
|
|
@@ -29,7 +29,7 @@ module ProcessExecuter
|
|
|
29
29
|
# | `CommandError` | A subclass of this error is raised when there is a problem executing a command. |
|
|
30
30
|
# | `FailedError` | Raised when the command exits with a non-zero exit status. |
|
|
31
31
|
# | `SignaledError` | Raised when the command is terminated as a result of receiving a signal. This could happen if the process is forcibly terminated or if there is a serious system error. |
|
|
32
|
-
# | `TimeoutError` | This is a specific type of `SignaledError` that is raised when the
|
|
32
|
+
# | `TimeoutError` | This is a specific type of `SignaledError` that is raised when the wait for the command is cut short by the configured `timeout_after` elapsing. The command is sent the SIGKILL signal, unless it had already exited in the moment between the timeout firing and the kill. |
|
|
33
33
|
# | `ProcessIOError` | Raised when an error was encountered reading or writing to the command's subprocess. |
|
|
34
34
|
# | `SpawnError` | Raised when the process could not execute. Check the `#cause` for the original exception from `Process.spawn`. |
|
|
35
35
|
#
|
|
@@ -139,7 +139,7 @@ module ProcessExecuter
|
|
|
139
139
|
#
|
|
140
140
|
class SignaledError < ProcessExecuter::CommandError; end
|
|
141
141
|
|
|
142
|
-
# Raised when the command
|
|
142
|
+
# Raised when the wait for the command is cut short by `timeout_after` elapsing
|
|
143
143
|
#
|
|
144
144
|
# @example
|
|
145
145
|
# begin
|
|
@@ -44,8 +44,9 @@ module ProcessExecuter
|
|
|
44
44
|
# a thread is created to read data written to the pipe. As data is read from the pipe,
|
|
45
45
|
# it is written to the destination provided in the MonitoredPipe initializer.
|
|
46
46
|
#
|
|
47
|
-
# If the destination raises an exception
|
|
48
|
-
#
|
|
47
|
+
# If the destination raises an exception (of any class, not just
|
|
48
|
+
# `StandardError`), the monitoring thread will exit, the pipe will be closed,
|
|
49
|
+
# and the exception will be saved in `#exception`.
|
|
49
50
|
#
|
|
50
51
|
# > **⚠️ WARNING**
|
|
51
52
|
# >
|
|
@@ -53,6 +54,21 @@ module ProcessExecuter
|
|
|
53
54
|
# read from the pipe and written to the destination, and (3) the monitoring thread is
|
|
54
55
|
# killed.
|
|
55
56
|
#
|
|
57
|
+
# ## File descriptor usage
|
|
58
|
+
#
|
|
59
|
+
# Each MonitoredPipe holds four file descriptors: two for the data pipe the
|
|
60
|
+
# subprocess writes to and two for an internal wake pipe that interrupts the
|
|
61
|
+
# monitoring thread when the pipe is closed. All four are held concurrently
|
|
62
|
+
# from construction until the pipe is closed, so a `ProcessExecuter.run` that
|
|
63
|
+
# captures stdout and stderr holds 8 pipe file descriptors for the duration
|
|
64
|
+
# of the subprocess.
|
|
65
|
+
#
|
|
66
|
+
# File descriptor limits are per-process. An application spawning many
|
|
67
|
+
# commands concurrently can hit the soft `RLIMIT_NOFILE` limit -- macOS
|
|
68
|
+
# defaults to 256 and Linux commonly to 1024. The workaround is to raise the
|
|
69
|
+
# limit in the spawning process, for example with
|
|
70
|
+
# `Process.setrlimit(:NOFILE, 10_240)` or `ulimit -n`.
|
|
71
|
+
#
|
|
56
72
|
# @example Collect pipe data into a StringIO object
|
|
57
73
|
# pipe_data = StringIO.new
|
|
58
74
|
# begin
|
|
@@ -93,6 +109,18 @@ module ProcessExecuter
|
|
|
93
109
|
class MonitoredPipe
|
|
94
110
|
include TrackOpenInstances
|
|
95
111
|
|
|
112
|
+
# The default number of seconds {#close} waits to drain remaining pipe data
|
|
113
|
+
#
|
|
114
|
+
# Draining normally finishes in well under a second: it only has to read
|
|
115
|
+
# whatever is still buffered in the pipe once every copy of the pipe's
|
|
116
|
+
# write fd is closed. The timeout exists so that a write fd inherited by a
|
|
117
|
+
# process outside this object's control (such as an orphaned descendant of
|
|
118
|
+
# a killed subprocess) cannot make {#close} block indefinitely.
|
|
119
|
+
#
|
|
120
|
+
# @return [Numeric]
|
|
121
|
+
#
|
|
122
|
+
DEFAULT_CLOSE_TIMEOUT = 10
|
|
123
|
+
|
|
96
124
|
# Create a new monitored pipe
|
|
97
125
|
#
|
|
98
126
|
# Creates an IO.pipe and starts a monitoring thread to read data written to the
|
|
@@ -121,29 +149,42 @@ module ProcessExecuter
|
|
|
121
149
|
#
|
|
122
150
|
def initialize(redirection_destination, chunk_size: 100_000)
|
|
123
151
|
@destination = Destinations.factory(redirection_destination)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
# Encoding::ASCII_8BIT, but it is a good practice to explicitly set the
|
|
135
|
-
# encoding.
|
|
136
|
-
pipe_reader.set_encoding(Encoding::ASCII_8BIT)
|
|
137
|
-
|
|
138
|
-
@state = :open
|
|
139
|
-
@thread = start_monitoring_thread
|
|
140
|
-
|
|
141
|
-
self.class.add_open_instance(self)
|
|
152
|
+
complete_initialization(chunk_size)
|
|
153
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
154
|
+
# The destination may hold resources (e.g. the File opened by
|
|
155
|
+
# Destinations::FilePath), so a failure partway through construction must
|
|
156
|
+
# close whatever was created so far -- the destination and, for each
|
|
157
|
+
# IO.pipe that succeeded, both pipe IOs -- or they leak. A failed
|
|
158
|
+
# initialize never returns a MonitoredPipe instance for the caller (or
|
|
159
|
+
# #close) to clean up.
|
|
160
|
+
[destination, pipe_reader, pipe_writer, wake_reader, wake_writer].each { |resource| resource&.close }
|
|
161
|
+
raise
|
|
142
162
|
end
|
|
143
163
|
|
|
144
164
|
# Set the state to `:closing` and wait for the state to be set to `:closed`
|
|
145
165
|
#
|
|
146
|
-
#
|
|
166
|
+
# A byte written to the internal wake pipe interrupts the monitoring
|
|
167
|
+
# thread's wait for pipe data; the thread then sees that the state has
|
|
168
|
+
# changed and closes the pipe.
|
|
169
|
+
#
|
|
170
|
+
# Remaining pipe data is drained to the destination for at most `timeout`
|
|
171
|
+
# seconds. The pipe only reaches EOF once every copy of its write fd is
|
|
172
|
+
# closed -- including copies inherited by processes outside this object's
|
|
173
|
+
# control -- so without a timeout this method could block indefinitely.
|
|
174
|
+
# When the timeout expires before EOF, the pipe is closed anyway,
|
|
175
|
+
# {#truncated?} returns `true`, and data still in the pipe is discarded.
|
|
176
|
+
#
|
|
177
|
+
# The timeout is one absolute deadline for the whole drain: time spent
|
|
178
|
+
# writing to the destination counts against it too. Only the waits for
|
|
179
|
+
# pipe data or EOF are cut short when the deadline passes, though -- a
|
|
180
|
+
# destination `#write` already in progress is never interrupted, so a
|
|
181
|
+
# destination that blocks can still delay this method past the timeout.
|
|
182
|
+
#
|
|
183
|
+
# An exception that escapes the monitoring thread's work is recorded in
|
|
184
|
+
# {#exception} by the monitoring thread itself before it terminates, so the
|
|
185
|
+
# `Thread#join` in this method never re-raises one. An exception raised at
|
|
186
|
+
# the join -- such as an `Interrupt` delivered to the calling thread -- is
|
|
187
|
+
# directed at the caller and propagates.
|
|
147
188
|
#
|
|
148
189
|
# @example
|
|
149
190
|
# data_collector = StringIO.new
|
|
@@ -154,15 +195,16 @@ module ProcessExecuter
|
|
|
154
195
|
# pipe.state #=> :closed
|
|
155
196
|
# data_collector.string #=> "Hello World"
|
|
156
197
|
#
|
|
198
|
+
# @param timeout [Numeric, nil] the number of seconds to spend draining
|
|
199
|
+
# remaining pipe data to the destination before giving up, or `nil` to
|
|
200
|
+
# wait without a time limit. The deadline is absolute -- time in the
|
|
201
|
+
# destination's `#write` counts against it -- but a write in progress is
|
|
202
|
+
# never interrupted, so a blocking destination can overrun it.
|
|
203
|
+
#
|
|
157
204
|
# @return [void]
|
|
158
205
|
#
|
|
159
|
-
def close
|
|
160
|
-
|
|
161
|
-
if state == :open
|
|
162
|
-
@state = :closing
|
|
163
|
-
condition_variable.wait(mutex) while @state != :closed
|
|
164
|
-
end
|
|
165
|
-
end
|
|
206
|
+
def close(timeout: DEFAULT_CLOSE_TIMEOUT)
|
|
207
|
+
initiate_close_and_wait_until_closed(timeout)
|
|
166
208
|
|
|
167
209
|
thread.join
|
|
168
210
|
destination.close
|
|
@@ -224,12 +266,35 @@ module ProcessExecuter
|
|
|
224
266
|
#
|
|
225
267
|
# @raise [IOError] if the pipe is not open
|
|
226
268
|
#
|
|
269
|
+
# The pipe is only checked before the write begins. If the destination
|
|
270
|
+
# raises (or another thread calls {#close}) while a large write is still in
|
|
271
|
+
# progress, the monitoring thread closes the pipe and the in-progress write
|
|
272
|
+
# fails with an `IOError` too.
|
|
273
|
+
#
|
|
227
274
|
def write(data)
|
|
228
|
-
mutex.
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
275
|
+
# The mutex is released before writing to the pipe. `pipe_writer.write`
|
|
276
|
+
# blocks once the OS pipe buffer is full, and it can only be unblocked by
|
|
277
|
+
# the monitoring thread draining the pipe. Holding the mutex across the
|
|
278
|
+
# write would stop the monitoring thread from taking the mutex in its own
|
|
279
|
+
# error path, deadlocking both threads.
|
|
280
|
+
mutex.synchronize { raise IOError, 'closed stream' unless state == :open }
|
|
281
|
+
|
|
282
|
+
pipe_writer.write(data)
|
|
283
|
+
rescue SystemCallError
|
|
284
|
+
# Engines disagree about how a write that is already blocked reacts to the
|
|
285
|
+
# monitoring thread closing the pipe. MRI and JRuby raise an IOError.
|
|
286
|
+
# TruffleRuby lets the write continue and fail at the system call, and which
|
|
287
|
+
# errno that is depends on the platform: EPIPE on macOS, EBADF on Linux.
|
|
288
|
+
#
|
|
289
|
+
# The monitoring thread is the only reader of this pipe, so any error from
|
|
290
|
+
# the operating system means the same thing the IOError does: the pipe went
|
|
291
|
+
# away mid-write. Report it as an IOError so #write has one documented
|
|
292
|
+
# contract on every supported engine. The original error is still available
|
|
293
|
+
# through Exception#cause.
|
|
294
|
+
#
|
|
295
|
+
# :nocov: only reached on engines that do not interrupt the blocked write
|
|
296
|
+
raise IOError, 'closed stream'
|
|
297
|
+
# :nocov:
|
|
233
298
|
end
|
|
234
299
|
|
|
235
300
|
# @!attribute [r]
|
|
@@ -293,6 +358,25 @@ module ProcessExecuter
|
|
|
293
358
|
#
|
|
294
359
|
attr_reader :exception
|
|
295
360
|
|
|
361
|
+
# Whether {#close} gave up draining the pipe before reaching EOF
|
|
362
|
+
#
|
|
363
|
+
# `true` when the close timeout expired before the pipe reached EOF: some
|
|
364
|
+
# copy of the pipe's write fd was still open (for instance, held by an
|
|
365
|
+
# orphaned descendant of a killed subprocess) or unread data remained, and
|
|
366
|
+
# what was left was discarded instead of being written to the destination.
|
|
367
|
+
# An expired timeout on a pipe with nothing left to drain closes normally
|
|
368
|
+
# and stays `false`.
|
|
369
|
+
#
|
|
370
|
+
# @example
|
|
371
|
+
# data_collector = StringIO.new
|
|
372
|
+
# pipe = ProcessExecuter::MonitoredPipe.new(data_collector)
|
|
373
|
+
# pipe.close
|
|
374
|
+
# pipe.truncated? #=> false
|
|
375
|
+
#
|
|
376
|
+
# @return [Boolean]
|
|
377
|
+
#
|
|
378
|
+
def truncated? = @truncated
|
|
379
|
+
|
|
296
380
|
# @!attribute [r]
|
|
297
381
|
#
|
|
298
382
|
# The thread that monitors the pipe
|
|
@@ -337,6 +421,32 @@ module ProcessExecuter
|
|
|
337
421
|
#
|
|
338
422
|
attr_reader :pipe_writer
|
|
339
423
|
|
|
424
|
+
# @!attribute [r]
|
|
425
|
+
#
|
|
426
|
+
# The read end of the internal wake pipe
|
|
427
|
+
#
|
|
428
|
+
# The monitoring thread waits on this IO (along with {#pipe_reader}) so
|
|
429
|
+
# that {#close} can interrupt its wait for pipe data.
|
|
430
|
+
#
|
|
431
|
+
# @return [IO]
|
|
432
|
+
#
|
|
433
|
+
# @api private
|
|
434
|
+
#
|
|
435
|
+
attr_reader :wake_reader
|
|
436
|
+
|
|
437
|
+
# @!attribute [r]
|
|
438
|
+
#
|
|
439
|
+
# The write end of the internal wake pipe
|
|
440
|
+
#
|
|
441
|
+
# {#close} writes a single byte to this IO -- at most once in the pipe's
|
|
442
|
+
# life -- to interrupt the monitoring thread's wait for pipe data.
|
|
443
|
+
#
|
|
444
|
+
# @return [IO]
|
|
445
|
+
#
|
|
446
|
+
# @api private
|
|
447
|
+
#
|
|
448
|
+
attr_reader :wake_writer
|
|
449
|
+
|
|
340
450
|
private
|
|
341
451
|
|
|
342
452
|
# @!attribute [r]
|
|
@@ -361,6 +471,82 @@ module ProcessExecuter
|
|
|
361
471
|
#
|
|
362
472
|
attr_reader :condition_variable
|
|
363
473
|
|
|
474
|
+
# Complete construction of the monitored pipe
|
|
475
|
+
#
|
|
476
|
+
# Performs every step of #initialize that can raise after the destination
|
|
477
|
+
# has been created: the compatibility check, creating the data and wake
|
|
478
|
+
# pipes, and starting the monitoring thread. #initialize cleans up the
|
|
479
|
+
# destination and any pipe IOs that were created if any of these steps
|
|
480
|
+
# fail.
|
|
481
|
+
#
|
|
482
|
+
# @param chunk_size [Integer] the size of the chunks to read from the pipe
|
|
483
|
+
# @return [void]
|
|
484
|
+
# @api private
|
|
485
|
+
def complete_initialization(chunk_size)
|
|
486
|
+
assert_destination_is_compatible_with_monitored_pipe
|
|
487
|
+
|
|
488
|
+
@mutex = Mutex.new
|
|
489
|
+
@condition_variable = ConditionVariable.new
|
|
490
|
+
@chunk_size = chunk_size
|
|
491
|
+
|
|
492
|
+
create_pipes
|
|
493
|
+
|
|
494
|
+
@state = :open
|
|
495
|
+
@truncated = false
|
|
496
|
+
@thread = start_monitoring_thread
|
|
497
|
+
|
|
498
|
+
self.class.add_open_instance(self)
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
# Create the data pipe and the internal wake pipe
|
|
502
|
+
#
|
|
503
|
+
# @return [void]
|
|
504
|
+
# @api private
|
|
505
|
+
def create_pipes
|
|
506
|
+
@pipe_reader, @pipe_writer = IO.pipe
|
|
507
|
+
|
|
508
|
+
# Set the encoding of the pipe reader to ASCII_8BIT. This is not strictly
|
|
509
|
+
# necessary since read_nonblock always returns a String where encoding is
|
|
510
|
+
# Encoding::ASCII_8BIT, but it is a good practice to explicitly set the
|
|
511
|
+
# encoding.
|
|
512
|
+
pipe_reader.set_encoding(Encoding::ASCII_8BIT)
|
|
513
|
+
|
|
514
|
+
# The wake pipe lets #close interrupt the monitoring thread, which
|
|
515
|
+
# otherwise blocks in IO.select waiting for pipe data (see #monitor_pipe)
|
|
516
|
+
@wake_reader, @wake_writer = IO.pipe
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
# Transition the state from `:open` to `:closing` and wait for `:closed`
|
|
520
|
+
#
|
|
521
|
+
# Implements the state-changing half of {#close}: under the mutex, record
|
|
522
|
+
# the close deadline, set the state to `:closing`, wake the monitoring
|
|
523
|
+
# thread, and wait for it to signal that the state reached `:closed`. A
|
|
524
|
+
# pipe that is not `:open` (already closing or closed) is left alone.
|
|
525
|
+
#
|
|
526
|
+
# @param timeout [Numeric, nil] seconds to spend draining remaining pipe
|
|
527
|
+
# data (see {#close}), or `nil` for no time limit
|
|
528
|
+
# @return [void]
|
|
529
|
+
# @api private
|
|
530
|
+
def initiate_close_and_wait_until_closed(timeout)
|
|
531
|
+
mutex.synchronize do
|
|
532
|
+
break unless state == :open
|
|
533
|
+
|
|
534
|
+
@close_deadline = timeout ? Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout : nil
|
|
535
|
+
@state = :closing
|
|
536
|
+
|
|
537
|
+
# The wake pipe is written at most once in its life: here, on the
|
|
538
|
+
# :open -> :closing transition. It is a one-shot signal, not a reusable
|
|
539
|
+
# channel, so the monitoring thread never needs to drain it (its loop
|
|
540
|
+
# never continues past this wakeup). The write needs no rescue: the
|
|
541
|
+
# monitoring thread closes the wake fds only after it publishes
|
|
542
|
+
# `@state = :closed` under this same mutex, so while this thread holds
|
|
543
|
+
# the mutex having observed :open, the wake fds are still open.
|
|
544
|
+
wake_writer.write('.')
|
|
545
|
+
|
|
546
|
+
condition_variable.wait(mutex) while @state != :closed
|
|
547
|
+
end
|
|
548
|
+
end
|
|
549
|
+
|
|
364
550
|
# Raise an error if the destination is not compatible with MonitoredPipe
|
|
365
551
|
# @return [void]
|
|
366
552
|
# @raise [ArgumentError] if the destination is not compatible with MonitoredPipe
|
|
@@ -372,38 +558,102 @@ module ProcessExecuter
|
|
|
372
558
|
end
|
|
373
559
|
|
|
374
560
|
# Start the thread to monitor the pipe and write data to the destination
|
|
561
|
+
#
|
|
562
|
+
# An exception that escapes {#monitor} is recorded in {#exception} (unless
|
|
563
|
+
# an exception is already recorded there) so that the thread never
|
|
564
|
+
# terminates holding an unhandled exception.
|
|
565
|
+
#
|
|
566
|
+
# The exception is recorded here, in the monitoring thread itself, rather
|
|
567
|
+
# than by rescuing around the `Thread#join` in {#close}, because a rescue
|
|
568
|
+
# at the join cannot reliably tell the two kinds of exception apart: a
|
|
569
|
+
# monitoring thread exception re-raised by `Thread#join` (which must be
|
|
570
|
+
# recorded) is indistinguishable from an async exception delivered to the
|
|
571
|
+
# calling thread at that same moment -- such as an `Interrupt` from Ctrl-C
|
|
572
|
+
# -- which must propagate. Checking `thread.alive?` in that rescue is racy
|
|
573
|
+
# since the thread can terminate between the exception being raised and
|
|
574
|
+
# the check. Recording at the source removes the ambiguity: the join can
|
|
575
|
+
# never re-raise a monitoring thread exception, so anything raised there
|
|
576
|
+
# is directed at the caller and propagates, while callers like
|
|
577
|
+
# `ProcessExecuter::Commands::Run` read {#exception} and report it as a
|
|
578
|
+
# {ProcessExecuter::ProcessIOError} with the original exception as its
|
|
579
|
+
# cause.
|
|
580
|
+
#
|
|
375
581
|
# @return [void]
|
|
376
582
|
# @api private
|
|
377
583
|
def start_monitoring_thread
|
|
378
584
|
Thread.new do
|
|
379
585
|
Thread.current.report_on_exception = false
|
|
380
586
|
Thread.current.abort_on_exception = false
|
|
381
|
-
|
|
587
|
+
begin
|
|
588
|
+
monitor
|
|
589
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
590
|
+
mutex.synchronize { @exception ||= e }
|
|
591
|
+
end
|
|
382
592
|
end
|
|
383
593
|
end
|
|
384
594
|
|
|
385
595
|
# Read data from the pipe until `#state` is changed to `:closing`
|
|
386
596
|
#
|
|
387
|
-
# The state is changed to `:
|
|
597
|
+
# The state is changed to `:closing` by calling `#close` (or by
|
|
598
|
+
# {#write_data} when the destination raises). The loop reads the state
|
|
599
|
+
# under the mutex: the writers of `@state` hold the mutex, and on engines
|
|
600
|
+
# with real parallelism (JRuby, TruffleRuby) an unsynchronized read has no
|
|
601
|
+
# memory barrier and thus no guarantee of seeing the transition.
|
|
602
|
+
#
|
|
603
|
+
# Before this method returns, state is set to `:closed`. This transition
|
|
604
|
+
# must happen even if closing the pipe raises, or a thread waiting in
|
|
605
|
+
# {#close} would block forever, so any exception raised by `#close_pipe` is
|
|
606
|
+
# saved to {#exception} instead of escaping the `ensure` block.
|
|
388
607
|
#
|
|
389
|
-
#
|
|
608
|
+
# The wake pipe is closed last, after `@state = :closed` is published
|
|
609
|
+
# under the mutex. The loop can also end without ever observing :closing
|
|
610
|
+
# (an exception raised by `#monitor_pipe` while the state is still :open),
|
|
611
|
+
# and in that case a concurrently arriving {#close} may still write the
|
|
612
|
+
# wake byte; this ordering guarantees the wake fds are open whenever
|
|
613
|
+
# {#close} observes :open under the mutex.
|
|
390
614
|
#
|
|
391
615
|
# @return [void]
|
|
392
616
|
# @api private
|
|
393
617
|
def monitor
|
|
394
|
-
monitor_pipe until state == :closing
|
|
618
|
+
monitor_pipe until mutex.synchronize { @state } == :closing
|
|
395
619
|
ensure
|
|
396
|
-
|
|
620
|
+
close_pipe_and_record_exception
|
|
397
621
|
mutex.synchronize do
|
|
398
622
|
@state = :closed
|
|
399
623
|
condition_variable.signal
|
|
400
624
|
end
|
|
625
|
+
close_wake_pipe
|
|
401
626
|
end
|
|
402
627
|
|
|
403
|
-
#
|
|
628
|
+
# Call `#close_pipe`, saving any exception it raises to {#exception}
|
|
629
|
+
#
|
|
630
|
+
# Rescues `Exception` (not just `StandardError`) so that the `ensure` block
|
|
631
|
+
# in {#monitor} always sets the state to `:closed` and signals the condition
|
|
632
|
+
# variable. The exception is recorded under the same mutex that guards the
|
|
633
|
+
# state transition, and an exception already recorded in {#exception} is not
|
|
634
|
+
# overwritten. This does not interfere with `Thread#kill` or `Thread#exit`,
|
|
635
|
+
# which terminate the thread through a mechanism that `rescue` cannot
|
|
636
|
+
# intercept; only exceptions that would otherwise escape are recorded.
|
|
637
|
+
#
|
|
638
|
+
# @return [void]
|
|
639
|
+
# @api private
|
|
640
|
+
def close_pipe_and_record_exception
|
|
641
|
+
close_pipe
|
|
642
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
643
|
+
mutex.synchronize { @exception ||= e }
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
# Read a chunk of data from the pipe or block until there is one to read
|
|
404
647
|
#
|
|
405
648
|
# Data read from the pipe is written to the destination.
|
|
406
649
|
#
|
|
650
|
+
# When the pipe has no data, block in IO.select (with no timeout, so an
|
|
651
|
+
# idle pipe costs no CPU) until either the pipe has data or {#close}
|
|
652
|
+
# writes its wake byte to the wake pipe. Either way this method returns
|
|
653
|
+
# and {#monitor} re-checks the state before calling it again, so the wake
|
|
654
|
+
# byte is never read: once it is written the state is already :closing and
|
|
655
|
+
# the loop exits.
|
|
656
|
+
#
|
|
407
657
|
# @return [void]
|
|
408
658
|
# @api private
|
|
409
659
|
def monitor_pipe
|
|
@@ -411,20 +661,28 @@ module ProcessExecuter
|
|
|
411
661
|
new_data = pipe_reader.read_nonblock(chunk_size)
|
|
412
662
|
write_data(new_data)
|
|
413
663
|
rescue IO::WaitReadable
|
|
414
|
-
|
|
664
|
+
IO.select([pipe_reader, wake_reader])
|
|
415
665
|
end
|
|
416
666
|
|
|
417
667
|
# Write the data read from the pipe to the destination
|
|
418
668
|
#
|
|
419
|
-
# If an exception is raised by a writer,
|
|
420
|
-
# so that the pipe can be closed.
|
|
669
|
+
# If an exception is raised by a writer, save it to {#exception} and set the
|
|
670
|
+
# state to `:closing` so that the pipe can be closed.
|
|
671
|
+
#
|
|
672
|
+
# Rescues `Exception` (not just `StandardError`) so that a destination
|
|
673
|
+
# raising, for instance, a `NoMemoryError` or a `SignalException` cannot
|
|
674
|
+
# kill the monitoring thread and leave {#close} blocked forever.
|
|
675
|
+
#
|
|
676
|
+
# Unlike {#close}, this error path needs no wake byte: it runs on the
|
|
677
|
+
# monitoring thread itself, and {#monitor}'s loop re-checks the state as
|
|
678
|
+
# soon as this method returns.
|
|
421
679
|
#
|
|
422
680
|
# @param data [String] the data read from the pipe
|
|
423
681
|
# @return [void]
|
|
424
682
|
# @api private
|
|
425
683
|
def write_data(data)
|
|
426
684
|
destination.write(data)
|
|
427
|
-
rescue
|
|
685
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
428
686
|
mutex.synchronize do
|
|
429
687
|
@exception = e
|
|
430
688
|
@state = :closing
|
|
@@ -441,10 +699,81 @@ module ProcessExecuter
|
|
|
441
699
|
|
|
442
700
|
# Read remaining data from pipe_reader (if any)
|
|
443
701
|
# If an exception was already raised by the last call to #write, then don't try to read remaining data
|
|
444
|
-
|
|
702
|
+
drain_pipe
|
|
445
703
|
|
|
446
704
|
# Close the read end of the pipe
|
|
447
705
|
pipe_reader.close
|
|
448
706
|
end
|
|
707
|
+
|
|
708
|
+
# Read remaining pipe data to the destination until EOF or the close deadline
|
|
709
|
+
#
|
|
710
|
+
# The pipe reaches EOF only once every copy of the write fd is closed,
|
|
711
|
+
# including copies inherited by processes this object knows nothing about
|
|
712
|
+
# (such as orphaned descendants of a killed subprocess). The deadline set
|
|
713
|
+
# by {#close} bounds the wait on such an fd: when it passes before EOF,
|
|
714
|
+
# draining stops and {#truncated?} becomes true. A `nil` deadline (a
|
|
715
|
+
# `close(timeout: nil)`, or the monitoring thread closing the pipe on its
|
|
716
|
+
# own after a destination exception) means no time limit.
|
|
717
|
+
#
|
|
718
|
+
# EOF is probed before the deadline is applied so that a pipe with nothing
|
|
719
|
+
# left to drain closes normally -- not as truncated -- even when the
|
|
720
|
+
# deadline has already passed (such as a `close(timeout: 0)`). Truncation
|
|
721
|
+
# is recorded only when the expired deadline abandons unread data or a
|
|
722
|
+
# still-open write fd.
|
|
723
|
+
#
|
|
724
|
+
# There is no need to poll: nothing outside this loop can end it, so each
|
|
725
|
+
# wait sleeps until the pipe has data or reaches EOF (both wake
|
|
726
|
+
# `wait_readable`), bounded by the time remaining before the deadline.
|
|
727
|
+
#
|
|
728
|
+
# The deadline bounds only the waits for pipe data or EOF. A call to
|
|
729
|
+
# {#write_data} runs the destination's `#write`, which is arbitrary user
|
|
730
|
+
# code that cannot safely be interrupted, so a destination that blocks can
|
|
731
|
+
# still hold this loop past the deadline; the deadline is applied again as
|
|
732
|
+
# soon as the write returns.
|
|
733
|
+
#
|
|
734
|
+
# @return [void]
|
|
735
|
+
# @api private
|
|
736
|
+
def drain_pipe
|
|
737
|
+
while exception.nil?
|
|
738
|
+
remaining_time = time_remaining_until_close_deadline
|
|
739
|
+
|
|
740
|
+
data = pipe_reader.read_nonblock(chunk_size, exception: false)
|
|
741
|
+
|
|
742
|
+
break if data.nil? # EOF: every copy of the pipe's write fd is closed
|
|
743
|
+
|
|
744
|
+
if remaining_time&.zero?
|
|
745
|
+
@truncated = true
|
|
746
|
+
break
|
|
747
|
+
end
|
|
748
|
+
|
|
749
|
+
data == :wait_readable ? pipe_reader.wait_readable(remaining_time) : write_data(data)
|
|
750
|
+
end
|
|
751
|
+
end
|
|
752
|
+
|
|
753
|
+
# Close both ends of the internal wake pipe
|
|
754
|
+
#
|
|
755
|
+
# Called from {#monitor}'s `ensure` block after `@state = :closed` is
|
|
756
|
+
# published, on every teardown path (a normal {#close}, a destination
|
|
757
|
+
# error, or an exception that ends the monitor loop). The `closed?` guards
|
|
758
|
+
# make it safe when a test helper has already closed the fds after killing
|
|
759
|
+
# the monitoring thread.
|
|
760
|
+
#
|
|
761
|
+
# @return [void]
|
|
762
|
+
# @api private
|
|
763
|
+
def close_wake_pipe
|
|
764
|
+
wake_writer.close unless wake_writer.closed?
|
|
765
|
+
wake_reader.close unless wake_reader.closed?
|
|
766
|
+
end
|
|
767
|
+
|
|
768
|
+
# The seconds left before the deadline set by {#close}
|
|
769
|
+
#
|
|
770
|
+
# @return [Numeric, nil] `nil` when no deadline is set (wait without a
|
|
771
|
+
# time limit), 0 when the deadline has passed
|
|
772
|
+
# @api private
|
|
773
|
+
def time_remaining_until_close_deadline
|
|
774
|
+
return nil if @close_deadline.nil?
|
|
775
|
+
|
|
776
|
+
[@close_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
|
|
777
|
+
end
|
|
449
778
|
end
|
|
450
779
|
end
|
|
@@ -151,20 +151,46 @@ module ProcessExecuter
|
|
|
151
151
|
#
|
|
152
152
|
# Subsequent hashes' values overwrite earlier ones for the same key.
|
|
153
153
|
#
|
|
154
|
+
# The merged options are checked the same way the constructor checks its
|
|
155
|
+
# options: unknown options and invalid option values raise a
|
|
156
|
+
# `ProcessExecuter::ArgumentError`. In that case, the current options
|
|
157
|
+
# object is left unchanged.
|
|
158
|
+
#
|
|
154
159
|
# @example
|
|
155
160
|
# options = MyOptions.new(option1: 'value1', option2: 'value2')
|
|
156
161
|
# h1 = { option2: 'new_value2' }
|
|
157
162
|
# h2 = { option3: 'value3' }
|
|
158
|
-
# options.merge!(h1, h2) => {option1: "value1", option2: "new_value2", option3: "value3"}
|
|
163
|
+
# options.merge!(h1, h2) # => options with {option1: "value1", option2: "new_value2", option3: "value3"}
|
|
164
|
+
#
|
|
165
|
+
# @example with an invalid option value
|
|
166
|
+
# options = MyOptions.new(option1: 'value1')
|
|
167
|
+
# begin
|
|
168
|
+
# options.merge!(option1: 1)
|
|
169
|
+
# rescue ProcessExecuter::ArgumentError => e
|
|
170
|
+
# e.message #=> "option1 must be a String but was 1"
|
|
171
|
+
# options.option1 #=> 'value1'
|
|
172
|
+
# end
|
|
159
173
|
#
|
|
160
|
-
# @param other_options_hashes [Array<Hash>] zero
|
|
174
|
+
# @param other_options_hashes [Array<Hash>] zero or more hashes to merge into the current options
|
|
161
175
|
#
|
|
162
176
|
# @return [self] the current options object with the merged options
|
|
163
177
|
#
|
|
178
|
+
# @raise [ProcessExecuter::ArgumentError] if the merged options contain an
|
|
179
|
+
# unknown option or an invalid option value
|
|
180
|
+
#
|
|
164
181
|
# @api public
|
|
165
182
|
#
|
|
166
183
|
def merge!(*other_options_hashes)
|
|
167
|
-
options_hash
|
|
184
|
+
original_options_hash = @options_hash
|
|
185
|
+
@options_hash = original_options_hash.dup.merge!(*other_options_hashes)
|
|
186
|
+
@errors = []
|
|
187
|
+
assert_no_unknown_options
|
|
188
|
+
validate_options
|
|
189
|
+
self
|
|
190
|
+
rescue ProcessExecuter::ArgumentError
|
|
191
|
+
@options_hash = original_options_hash
|
|
192
|
+
@errors = []
|
|
193
|
+
raise
|
|
168
194
|
end
|
|
169
195
|
|
|
170
196
|
# Returns a new options object formed by merging self with each of other_hashes
|
|
@@ -181,6 +207,9 @@ module ProcessExecuter
|
|
|
181
207
|
#
|
|
182
208
|
# @return [self.class]
|
|
183
209
|
#
|
|
210
|
+
# @raise [ProcessExecuter::ArgumentError] if the merged options contain an
|
|
211
|
+
# unknown option or an invalid option value
|
|
212
|
+
#
|
|
184
213
|
def merge(*other_options_hashes)
|
|
185
214
|
merged_options = other_options_hashes.reduce(options_hash, :merge)
|
|
186
215
|
self.class.new(**merged_options)
|