process_executer 4.0.4 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.editorconfig +10 -0
- data/.release-please-manifest.json +1 -1
- data/.rubocop.yml +13 -0
- data/.vscode/extensions.json +3 -0
- data/CHANGELOG.md +56 -0
- data/README.md +20 -0
- data/Rakefile +3 -0
- data/lib/process_executer/commands/run.rb +163 -9
- data/lib/process_executer/commands/run_with_capture.rb +89 -35
- data/lib/process_executer/commands/spawn_with_timeout.rb +215 -16
- data/lib/process_executer/destinations/monitored_pipe.rb +7 -4
- data/lib/process_executer/errors.rb +2 -2
- data/lib/process_executer/monitored_pipe.rb +385 -47
- data/lib/process_executer/options/base.rb +48 -7
- data/lib/process_executer/options/run_with_capture_options.rb +92 -61
- data/lib/process_executer/options/spawn_options.rb +13 -0
- data/lib/process_executer/result.rb +13 -2
- data/lib/process_executer/version.rb +1 -1
- data/lib/process_executer.rb +34 -2
- data/process_spawn_test/README.md +38 -1
- data/process_spawn_test/spec/spec_helper.rb +18 -0
- data/process_spawn_test/spec/test_spec.rb +21 -0
- metadata +6 -4
|
@@ -44,8 +44,10 @@ module ProcessExecuter
|
|
|
44
44
|
# a thread is created to read data written to the pipe. As data is read from the pipe,
|
|
45
45
|
# it is written to the destination provided in the MonitoredPipe initializer.
|
|
46
46
|
#
|
|
47
|
-
# If
|
|
48
|
-
#
|
|
47
|
+
# If an exception (of any class, not just `StandardError`) is raised while
|
|
48
|
+
# collecting output -- by the destination's `#write`, by the monitor loop,
|
|
49
|
+
# or by pipe cleanup -- the monitoring thread exits, the pipe is closed, and
|
|
50
|
+
# the exception is saved in {#exception}.
|
|
49
51
|
#
|
|
50
52
|
# > **⚠️ WARNING**
|
|
51
53
|
# >
|
|
@@ -53,6 +55,21 @@ module ProcessExecuter
|
|
|
53
55
|
# read from the pipe and written to the destination, and (3) the monitoring thread is
|
|
54
56
|
# killed.
|
|
55
57
|
#
|
|
58
|
+
# ## File descriptor usage
|
|
59
|
+
#
|
|
60
|
+
# Each MonitoredPipe holds four file descriptors: two for the data pipe the
|
|
61
|
+
# subprocess writes to and two for an internal wake pipe that interrupts the
|
|
62
|
+
# monitoring thread when the pipe is closed. All four are held concurrently
|
|
63
|
+
# from construction until the pipe is closed, so a `ProcessExecuter.run` that
|
|
64
|
+
# captures stdout and stderr holds 8 pipe file descriptors for the duration
|
|
65
|
+
# of the subprocess.
|
|
66
|
+
#
|
|
67
|
+
# File descriptor limits are per-process. An application spawning many
|
|
68
|
+
# commands concurrently can hit the soft `RLIMIT_NOFILE` limit -- macOS
|
|
69
|
+
# defaults to 256 and Linux commonly to 1024. The workaround is to raise the
|
|
70
|
+
# limit in the spawning process, for example with
|
|
71
|
+
# `Process.setrlimit(:NOFILE, 10_240)` or `ulimit -n`.
|
|
72
|
+
#
|
|
56
73
|
# @example Collect pipe data into a StringIO object
|
|
57
74
|
# pipe_data = StringIO.new
|
|
58
75
|
# begin
|
|
@@ -93,6 +110,18 @@ module ProcessExecuter
|
|
|
93
110
|
class MonitoredPipe
|
|
94
111
|
include TrackOpenInstances
|
|
95
112
|
|
|
113
|
+
# The default number of seconds {#close} waits to drain remaining pipe data
|
|
114
|
+
#
|
|
115
|
+
# Draining normally finishes in well under a second: it only has to read
|
|
116
|
+
# whatever is still buffered in the pipe once every copy of the pipe's
|
|
117
|
+
# write fd is closed. The timeout exists so that a write fd inherited by a
|
|
118
|
+
# process outside this object's control (such as an orphaned descendant of
|
|
119
|
+
# a killed subprocess) cannot make {#close} block indefinitely.
|
|
120
|
+
#
|
|
121
|
+
# @return [Numeric]
|
|
122
|
+
#
|
|
123
|
+
DEFAULT_CLOSE_TIMEOUT = 10
|
|
124
|
+
|
|
96
125
|
# Create a new monitored pipe
|
|
97
126
|
#
|
|
98
127
|
# Creates an IO.pipe and starts a monitoring thread to read data written to the
|
|
@@ -121,29 +150,42 @@ module ProcessExecuter
|
|
|
121
150
|
#
|
|
122
151
|
def initialize(redirection_destination, chunk_size: 100_000)
|
|
123
152
|
@destination = Destinations.factory(redirection_destination)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
# Encoding::ASCII_8BIT, but it is a good practice to explicitly set the
|
|
135
|
-
# encoding.
|
|
136
|
-
pipe_reader.set_encoding(Encoding::ASCII_8BIT)
|
|
137
|
-
|
|
138
|
-
@state = :open
|
|
139
|
-
@thread = start_monitoring_thread
|
|
140
|
-
|
|
141
|
-
self.class.add_open_instance(self)
|
|
153
|
+
complete_initialization(chunk_size)
|
|
154
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
155
|
+
# The destination may hold resources (e.g. the File opened by
|
|
156
|
+
# Destinations::FilePath), so a failure partway through construction must
|
|
157
|
+
# close whatever was created so far -- the destination and, for each
|
|
158
|
+
# IO.pipe that succeeded, both pipe IOs -- or they leak. A failed
|
|
159
|
+
# initialize never returns a MonitoredPipe instance for the caller (or
|
|
160
|
+
# #close) to clean up.
|
|
161
|
+
[destination, pipe_reader, pipe_writer, wake_reader, wake_writer].each { |resource| resource&.close }
|
|
162
|
+
raise
|
|
142
163
|
end
|
|
143
164
|
|
|
144
165
|
# Set the state to `:closing` and wait for the state to be set to `:closed`
|
|
145
166
|
#
|
|
146
|
-
#
|
|
167
|
+
# A byte written to the internal wake pipe interrupts the monitoring
|
|
168
|
+
# thread's wait for pipe data; the thread then sees that the state has
|
|
169
|
+
# changed and closes the pipe.
|
|
170
|
+
#
|
|
171
|
+
# Remaining pipe data is drained to the destination for at most `timeout`
|
|
172
|
+
# seconds. The pipe only reaches EOF once every copy of its write fd is
|
|
173
|
+
# closed -- including copies inherited by processes outside this object's
|
|
174
|
+
# control -- so without a timeout this method could block indefinitely.
|
|
175
|
+
# When the timeout expires before EOF, the pipe is closed anyway,
|
|
176
|
+
# {#truncated?} returns `true`, and data still in the pipe is discarded.
|
|
177
|
+
#
|
|
178
|
+
# The timeout is one absolute deadline for the whole drain: time spent
|
|
179
|
+
# writing to the destination counts against it too. Only the waits for
|
|
180
|
+
# pipe data or EOF are cut short when the deadline passes, though -- a
|
|
181
|
+
# destination `#write` already in progress is never interrupted, so a
|
|
182
|
+
# destination that blocks can still delay this method past the timeout.
|
|
183
|
+
#
|
|
184
|
+
# An exception that escapes the monitoring thread's work is recorded in
|
|
185
|
+
# {#exception} by the monitoring thread itself before it terminates, so the
|
|
186
|
+
# `Thread#join` in this method never re-raises one. An exception raised at
|
|
187
|
+
# the join -- such as an `Interrupt` delivered to the calling thread -- is
|
|
188
|
+
# directed at the caller and propagates.
|
|
147
189
|
#
|
|
148
190
|
# @example
|
|
149
191
|
# data_collector = StringIO.new
|
|
@@ -154,15 +196,16 @@ module ProcessExecuter
|
|
|
154
196
|
# pipe.state #=> :closed
|
|
155
197
|
# data_collector.string #=> "Hello World"
|
|
156
198
|
#
|
|
199
|
+
# @param timeout [Numeric, nil] the number of seconds to spend draining
|
|
200
|
+
# remaining pipe data to the destination before giving up, or `nil` to
|
|
201
|
+
# wait without a time limit. The deadline is absolute -- time in the
|
|
202
|
+
# destination's `#write` counts against it -- but a write in progress is
|
|
203
|
+
# never interrupted, so a blocking destination can overrun it.
|
|
204
|
+
#
|
|
157
205
|
# @return [void]
|
|
158
206
|
#
|
|
159
|
-
def close
|
|
160
|
-
|
|
161
|
-
if state == :open
|
|
162
|
-
@state = :closing
|
|
163
|
-
condition_variable.wait(mutex) while @state != :closed
|
|
164
|
-
end
|
|
165
|
-
end
|
|
207
|
+
def close(timeout: DEFAULT_CLOSE_TIMEOUT)
|
|
208
|
+
initiate_close_and_wait_until_closed(timeout)
|
|
166
209
|
|
|
167
210
|
thread.join
|
|
168
211
|
destination.close
|
|
@@ -224,12 +267,35 @@ module ProcessExecuter
|
|
|
224
267
|
#
|
|
225
268
|
# @raise [IOError] if the pipe is not open
|
|
226
269
|
#
|
|
270
|
+
# The pipe is only checked before the write begins. If the destination
|
|
271
|
+
# raises (or another thread calls {#close}) while a large write is still in
|
|
272
|
+
# progress, the monitoring thread closes the pipe and the in-progress write
|
|
273
|
+
# fails with an `IOError` too.
|
|
274
|
+
#
|
|
227
275
|
def write(data)
|
|
228
|
-
mutex.
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
276
|
+
# The mutex is released before writing to the pipe. `pipe_writer.write`
|
|
277
|
+
# blocks once the OS pipe buffer is full, and it can only be unblocked by
|
|
278
|
+
# the monitoring thread draining the pipe. Holding the mutex across the
|
|
279
|
+
# write would stop the monitoring thread from taking the mutex in its own
|
|
280
|
+
# error path, deadlocking both threads.
|
|
281
|
+
mutex.synchronize { raise IOError, 'closed stream' unless state == :open }
|
|
282
|
+
|
|
283
|
+
pipe_writer.write(data)
|
|
284
|
+
rescue SystemCallError
|
|
285
|
+
# Engines disagree about how a write that is already blocked reacts to the
|
|
286
|
+
# monitoring thread closing the pipe. MRI and JRuby raise an IOError.
|
|
287
|
+
# TruffleRuby lets the write continue and fail at the system call, and which
|
|
288
|
+
# errno that is depends on the platform: EPIPE on macOS, EBADF on Linux.
|
|
289
|
+
#
|
|
290
|
+
# The monitoring thread is the only reader of this pipe, so any error from
|
|
291
|
+
# the operating system means the same thing the IOError does: the pipe went
|
|
292
|
+
# away mid-write. Report it as an IOError so #write has one documented
|
|
293
|
+
# contract on every supported engine. The original error is still available
|
|
294
|
+
# through Exception#cause.
|
|
295
|
+
#
|
|
296
|
+
# :nocov: only reached on engines that do not interrupt the blocked write
|
|
297
|
+
raise IOError, 'closed stream'
|
|
298
|
+
# :nocov:
|
|
233
299
|
end
|
|
234
300
|
|
|
235
301
|
# @!attribute [r]
|
|
@@ -282,17 +348,44 @@ module ProcessExecuter
|
|
|
282
348
|
|
|
283
349
|
# @!attribute [r]
|
|
284
350
|
#
|
|
285
|
-
# The exception
|
|
351
|
+
# The first exception recorded while collecting output
|
|
286
352
|
#
|
|
287
|
-
#
|
|
353
|
+
# Any failure while collecting output is recorded here, not only a
|
|
354
|
+
# destination error: the destination's `#write` raising, the monitor loop
|
|
355
|
+
# raising, or pipe cleanup raising. `nil` if no exception was raised.
|
|
356
|
+
#
|
|
357
|
+
# When more than one exception is raised, the first one *recorded* wins
|
|
358
|
+
# and the rest are discarded. Recording order has one corner: when the
|
|
359
|
+
# monitor loop raises and pipe cleanup then also raises, the cleanup
|
|
360
|
+
# error is the one recorded, because the monitor-loop exception is still
|
|
361
|
+
# in flight while cleanup runs and reaches its recording site last.
|
|
288
362
|
#
|
|
289
363
|
# @example
|
|
290
364
|
# pipe.exception #=> nil
|
|
291
365
|
#
|
|
292
|
-
# @return [Exception, nil] the
|
|
366
|
+
# @return [Exception, nil] the first recorded exception or `nil` if no exception was raised
|
|
293
367
|
#
|
|
294
368
|
attr_reader :exception
|
|
295
369
|
|
|
370
|
+
# Whether {#close} gave up draining the pipe before reaching EOF
|
|
371
|
+
#
|
|
372
|
+
# `true` when the close timeout expired before the pipe reached EOF: some
|
|
373
|
+
# copy of the pipe's write fd was still open (for instance, held by an
|
|
374
|
+
# orphaned descendant of a killed subprocess) or unread data remained, and
|
|
375
|
+
# what was left was discarded instead of being written to the destination.
|
|
376
|
+
# An expired timeout on a pipe with nothing left to drain closes normally
|
|
377
|
+
# and stays `false`.
|
|
378
|
+
#
|
|
379
|
+
# @example
|
|
380
|
+
# data_collector = StringIO.new
|
|
381
|
+
# pipe = ProcessExecuter::MonitoredPipe.new(data_collector)
|
|
382
|
+
# pipe.close
|
|
383
|
+
# pipe.truncated? #=> false
|
|
384
|
+
#
|
|
385
|
+
# @return [Boolean]
|
|
386
|
+
#
|
|
387
|
+
def truncated? = @truncated
|
|
388
|
+
|
|
296
389
|
# @!attribute [r]
|
|
297
390
|
#
|
|
298
391
|
# The thread that monitors the pipe
|
|
@@ -337,6 +430,32 @@ module ProcessExecuter
|
|
|
337
430
|
#
|
|
338
431
|
attr_reader :pipe_writer
|
|
339
432
|
|
|
433
|
+
# @!attribute [r]
|
|
434
|
+
#
|
|
435
|
+
# The read end of the internal wake pipe
|
|
436
|
+
#
|
|
437
|
+
# The monitoring thread waits on this IO (along with {#pipe_reader}) so
|
|
438
|
+
# that {#close} can interrupt its wait for pipe data.
|
|
439
|
+
#
|
|
440
|
+
# @return [IO]
|
|
441
|
+
#
|
|
442
|
+
# @api private
|
|
443
|
+
#
|
|
444
|
+
attr_reader :wake_reader
|
|
445
|
+
|
|
446
|
+
# @!attribute [r]
|
|
447
|
+
#
|
|
448
|
+
# The write end of the internal wake pipe
|
|
449
|
+
#
|
|
450
|
+
# {#close} writes a single byte to this IO -- at most once in the pipe's
|
|
451
|
+
# life -- to interrupt the monitoring thread's wait for pipe data.
|
|
452
|
+
#
|
|
453
|
+
# @return [IO]
|
|
454
|
+
#
|
|
455
|
+
# @api private
|
|
456
|
+
#
|
|
457
|
+
attr_reader :wake_writer
|
|
458
|
+
|
|
340
459
|
private
|
|
341
460
|
|
|
342
461
|
# @!attribute [r]
|
|
@@ -361,6 +480,82 @@ module ProcessExecuter
|
|
|
361
480
|
#
|
|
362
481
|
attr_reader :condition_variable
|
|
363
482
|
|
|
483
|
+
# Complete construction of the monitored pipe
|
|
484
|
+
#
|
|
485
|
+
# Performs every step of #initialize that can raise after the destination
|
|
486
|
+
# has been created: the compatibility check, creating the data and wake
|
|
487
|
+
# pipes, and starting the monitoring thread. #initialize cleans up the
|
|
488
|
+
# destination and any pipe IOs that were created if any of these steps
|
|
489
|
+
# fail.
|
|
490
|
+
#
|
|
491
|
+
# @param chunk_size [Integer] the size of the chunks to read from the pipe
|
|
492
|
+
# @return [void]
|
|
493
|
+
# @api private
|
|
494
|
+
def complete_initialization(chunk_size)
|
|
495
|
+
assert_destination_is_compatible_with_monitored_pipe
|
|
496
|
+
|
|
497
|
+
@mutex = Mutex.new
|
|
498
|
+
@condition_variable = ConditionVariable.new
|
|
499
|
+
@chunk_size = chunk_size
|
|
500
|
+
|
|
501
|
+
create_pipes
|
|
502
|
+
|
|
503
|
+
@state = :open
|
|
504
|
+
@truncated = false
|
|
505
|
+
@thread = start_monitoring_thread
|
|
506
|
+
|
|
507
|
+
self.class.add_open_instance(self)
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# Create the data pipe and the internal wake pipe
|
|
511
|
+
#
|
|
512
|
+
# @return [void]
|
|
513
|
+
# @api private
|
|
514
|
+
def create_pipes
|
|
515
|
+
@pipe_reader, @pipe_writer = IO.pipe
|
|
516
|
+
|
|
517
|
+
# Set the encoding of the pipe reader to ASCII_8BIT. This is not strictly
|
|
518
|
+
# necessary since read_nonblock always returns a String where encoding is
|
|
519
|
+
# Encoding::ASCII_8BIT, but it is a good practice to explicitly set the
|
|
520
|
+
# encoding.
|
|
521
|
+
pipe_reader.set_encoding(Encoding::ASCII_8BIT)
|
|
522
|
+
|
|
523
|
+
# The wake pipe lets #close interrupt the monitoring thread, which
|
|
524
|
+
# otherwise blocks in IO.select waiting for pipe data (see #monitor_pipe)
|
|
525
|
+
@wake_reader, @wake_writer = IO.pipe
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
# Transition the state from `:open` to `:closing` and wait for `:closed`
|
|
529
|
+
#
|
|
530
|
+
# Implements the state-changing half of {#close}: under the mutex, record
|
|
531
|
+
# the close deadline, set the state to `:closing`, wake the monitoring
|
|
532
|
+
# thread, and wait for it to signal that the state reached `:closed`. A
|
|
533
|
+
# pipe that is not `:open` (already closing or closed) is left alone.
|
|
534
|
+
#
|
|
535
|
+
# @param timeout [Numeric, nil] seconds to spend draining remaining pipe
|
|
536
|
+
# data (see {#close}), or `nil` for no time limit
|
|
537
|
+
# @return [void]
|
|
538
|
+
# @api private
|
|
539
|
+
def initiate_close_and_wait_until_closed(timeout)
|
|
540
|
+
mutex.synchronize do
|
|
541
|
+
break unless state == :open
|
|
542
|
+
|
|
543
|
+
@close_deadline = timeout ? Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout : nil
|
|
544
|
+
@state = :closing
|
|
545
|
+
|
|
546
|
+
# The wake pipe is written at most once in its life: here, on the
|
|
547
|
+
# :open -> :closing transition. It is a one-shot signal, not a reusable
|
|
548
|
+
# channel, so the monitoring thread never needs to drain it (its loop
|
|
549
|
+
# never continues past this wakeup). The write needs no rescue: the
|
|
550
|
+
# monitoring thread closes the wake fds only after it publishes
|
|
551
|
+
# `@state = :closed` under this same mutex, so while this thread holds
|
|
552
|
+
# the mutex having observed :open, the wake fds are still open.
|
|
553
|
+
wake_writer.write('.')
|
|
554
|
+
|
|
555
|
+
condition_variable.wait(mutex) while @state != :closed
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
|
|
364
559
|
# Raise an error if the destination is not compatible with MonitoredPipe
|
|
365
560
|
# @return [void]
|
|
366
561
|
# @raise [ArgumentError] if the destination is not compatible with MonitoredPipe
|
|
@@ -372,38 +567,102 @@ module ProcessExecuter
|
|
|
372
567
|
end
|
|
373
568
|
|
|
374
569
|
# Start the thread to monitor the pipe and write data to the destination
|
|
570
|
+
#
|
|
571
|
+
# An exception that escapes {#monitor} is recorded in {#exception} (unless
|
|
572
|
+
# an exception is already recorded there) so that the thread never
|
|
573
|
+
# terminates holding an unhandled exception.
|
|
574
|
+
#
|
|
575
|
+
# The exception is recorded here, in the monitoring thread itself, rather
|
|
576
|
+
# than by rescuing around the `Thread#join` in {#close}, because a rescue
|
|
577
|
+
# at the join cannot reliably tell the two kinds of exception apart: a
|
|
578
|
+
# monitoring thread exception re-raised by `Thread#join` (which must be
|
|
579
|
+
# recorded) is indistinguishable from an async exception delivered to the
|
|
580
|
+
# calling thread at that same moment -- such as an `Interrupt` from Ctrl-C
|
|
581
|
+
# -- which must propagate. Checking `thread.alive?` in that rescue is racy
|
|
582
|
+
# since the thread can terminate between the exception being raised and
|
|
583
|
+
# the check. Recording at the source removes the ambiguity: the join can
|
|
584
|
+
# never re-raise a monitoring thread exception, so anything raised there
|
|
585
|
+
# is directed at the caller and propagates, while callers like
|
|
586
|
+
# `ProcessExecuter::Commands::Run` read {#exception} and report it as a
|
|
587
|
+
# {ProcessExecuter::ProcessIOError} with the original exception as its
|
|
588
|
+
# cause.
|
|
589
|
+
#
|
|
375
590
|
# @return [void]
|
|
376
591
|
# @api private
|
|
377
592
|
def start_monitoring_thread
|
|
378
593
|
Thread.new do
|
|
379
594
|
Thread.current.report_on_exception = false
|
|
380
595
|
Thread.current.abort_on_exception = false
|
|
381
|
-
|
|
596
|
+
begin
|
|
597
|
+
monitor
|
|
598
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
599
|
+
mutex.synchronize { @exception ||= e }
|
|
600
|
+
end
|
|
382
601
|
end
|
|
383
602
|
end
|
|
384
603
|
|
|
385
604
|
# Read data from the pipe until `#state` is changed to `:closing`
|
|
386
605
|
#
|
|
387
|
-
# The state is changed to `:
|
|
606
|
+
# The state is changed to `:closing` by calling `#close` (or by
|
|
607
|
+
# {#write_data} when the destination raises). The loop reads the state
|
|
608
|
+
# under the mutex: the writers of `@state` hold the mutex, and on engines
|
|
609
|
+
# with real parallelism (JRuby, TruffleRuby) an unsynchronized read has no
|
|
610
|
+
# memory barrier and thus no guarantee of seeing the transition.
|
|
611
|
+
#
|
|
612
|
+
# Before this method returns, state is set to `:closed`. This transition
|
|
613
|
+
# must happen even if closing the pipe raises, or a thread waiting in
|
|
614
|
+
# {#close} would block forever, so any exception raised by `#close_pipe` is
|
|
615
|
+
# saved to {#exception} instead of escaping the `ensure` block.
|
|
388
616
|
#
|
|
389
|
-
#
|
|
617
|
+
# The wake pipe is closed last, after `@state = :closed` is published
|
|
618
|
+
# under the mutex. The loop can also end without ever observing :closing
|
|
619
|
+
# (an exception raised by `#monitor_pipe` while the state is still :open),
|
|
620
|
+
# and in that case a concurrently arriving {#close} may still write the
|
|
621
|
+
# wake byte; this ordering guarantees the wake fds are open whenever
|
|
622
|
+
# {#close} observes :open under the mutex.
|
|
390
623
|
#
|
|
391
624
|
# @return [void]
|
|
392
625
|
# @api private
|
|
393
626
|
def monitor
|
|
394
|
-
monitor_pipe until state == :closing
|
|
627
|
+
monitor_pipe until mutex.synchronize { @state } == :closing
|
|
395
628
|
ensure
|
|
396
|
-
|
|
629
|
+
close_pipe_and_record_exception
|
|
397
630
|
mutex.synchronize do
|
|
398
631
|
@state = :closed
|
|
399
632
|
condition_variable.signal
|
|
400
633
|
end
|
|
634
|
+
close_wake_pipe
|
|
401
635
|
end
|
|
402
636
|
|
|
403
|
-
#
|
|
637
|
+
# Call `#close_pipe`, saving any exception it raises to {#exception}
|
|
638
|
+
#
|
|
639
|
+
# Rescues `Exception` (not just `StandardError`) so that the `ensure` block
|
|
640
|
+
# in {#monitor} always sets the state to `:closed` and signals the condition
|
|
641
|
+
# variable. The exception is recorded under the same mutex that guards the
|
|
642
|
+
# state transition, and an exception already recorded in {#exception} is not
|
|
643
|
+
# overwritten. This does not interfere with `Thread#kill` or `Thread#exit`,
|
|
644
|
+
# which terminate the thread through a mechanism that `rescue` cannot
|
|
645
|
+
# intercept; only exceptions that would otherwise escape are recorded.
|
|
646
|
+
#
|
|
647
|
+
# @return [void]
|
|
648
|
+
# @api private
|
|
649
|
+
def close_pipe_and_record_exception
|
|
650
|
+
close_pipe
|
|
651
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
652
|
+
mutex.synchronize { @exception ||= e }
|
|
653
|
+
end
|
|
654
|
+
|
|
655
|
+
# Read a chunk of data from the pipe or block until there is one to read
|
|
404
656
|
#
|
|
405
657
|
# Data read from the pipe is written to the destination.
|
|
406
658
|
#
|
|
659
|
+
# When the pipe has no data, block in IO.select (with no timeout, so an
|
|
660
|
+
# idle pipe costs no CPU) until either the pipe has data or {#close}
|
|
661
|
+
# writes its wake byte to the wake pipe. Either way this method returns
|
|
662
|
+
# and {#monitor} re-checks the state before calling it again, so the wake
|
|
663
|
+
# byte is never read: once it is written the state is already :closing and
|
|
664
|
+
# the loop exits.
|
|
665
|
+
#
|
|
407
666
|
# @return [void]
|
|
408
667
|
# @api private
|
|
409
668
|
def monitor_pipe
|
|
@@ -411,20 +670,28 @@ module ProcessExecuter
|
|
|
411
670
|
new_data = pipe_reader.read_nonblock(chunk_size)
|
|
412
671
|
write_data(new_data)
|
|
413
672
|
rescue IO::WaitReadable
|
|
414
|
-
|
|
673
|
+
IO.select([pipe_reader, wake_reader])
|
|
415
674
|
end
|
|
416
675
|
|
|
417
676
|
# Write the data read from the pipe to the destination
|
|
418
677
|
#
|
|
419
|
-
# If an exception is raised by a writer,
|
|
420
|
-
# so that the pipe can be closed.
|
|
678
|
+
# If an exception is raised by a writer, save it to {#exception} and set the
|
|
679
|
+
# state to `:closing` so that the pipe can be closed.
|
|
680
|
+
#
|
|
681
|
+
# Rescues `Exception` (not just `StandardError`) so that a destination
|
|
682
|
+
# raising, for instance, a `NoMemoryError` or a `SignalException` cannot
|
|
683
|
+
# kill the monitoring thread and leave {#close} blocked forever.
|
|
684
|
+
#
|
|
685
|
+
# Unlike {#close}, this error path needs no wake byte: it runs on the
|
|
686
|
+
# monitoring thread itself, and {#monitor}'s loop re-checks the state as
|
|
687
|
+
# soon as this method returns.
|
|
421
688
|
#
|
|
422
689
|
# @param data [String] the data read from the pipe
|
|
423
690
|
# @return [void]
|
|
424
691
|
# @api private
|
|
425
692
|
def write_data(data)
|
|
426
693
|
destination.write(data)
|
|
427
|
-
rescue
|
|
694
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
428
695
|
mutex.synchronize do
|
|
429
696
|
@exception = e
|
|
430
697
|
@state = :closing
|
|
@@ -441,10 +708,81 @@ module ProcessExecuter
|
|
|
441
708
|
|
|
442
709
|
# Read remaining data from pipe_reader (if any)
|
|
443
710
|
# If an exception was already raised by the last call to #write, then don't try to read remaining data
|
|
444
|
-
|
|
711
|
+
drain_pipe
|
|
445
712
|
|
|
446
713
|
# Close the read end of the pipe
|
|
447
714
|
pipe_reader.close
|
|
448
715
|
end
|
|
716
|
+
|
|
717
|
+
# Read remaining pipe data to the destination until EOF or the close deadline
|
|
718
|
+
#
|
|
719
|
+
# The pipe reaches EOF only once every copy of the write fd is closed,
|
|
720
|
+
# including copies inherited by processes this object knows nothing about
|
|
721
|
+
# (such as orphaned descendants of a killed subprocess). The deadline set
|
|
722
|
+
# by {#close} bounds the wait on such an fd: when it passes before EOF,
|
|
723
|
+
# draining stops and {#truncated?} becomes true. A `nil` deadline (a
|
|
724
|
+
# `close(timeout: nil)`, or the monitoring thread closing the pipe on its
|
|
725
|
+
# own after a destination exception) means no time limit.
|
|
726
|
+
#
|
|
727
|
+
# EOF is probed before the deadline is applied so that a pipe with nothing
|
|
728
|
+
# left to drain closes normally -- not as truncated -- even when the
|
|
729
|
+
# deadline has already passed (such as a `close(timeout: 0)`). Truncation
|
|
730
|
+
# is recorded only when the expired deadline abandons unread data or a
|
|
731
|
+
# still-open write fd.
|
|
732
|
+
#
|
|
733
|
+
# There is no need to poll: nothing outside this loop can end it, so each
|
|
734
|
+
# wait sleeps until the pipe has data or reaches EOF (both wake
|
|
735
|
+
# `wait_readable`), bounded by the time remaining before the deadline.
|
|
736
|
+
#
|
|
737
|
+
# The deadline bounds only the waits for pipe data or EOF. A call to
|
|
738
|
+
# {#write_data} runs the destination's `#write`, which is arbitrary user
|
|
739
|
+
# code that cannot safely be interrupted, so a destination that blocks can
|
|
740
|
+
# still hold this loop past the deadline; the deadline is applied again as
|
|
741
|
+
# soon as the write returns.
|
|
742
|
+
#
|
|
743
|
+
# @return [void]
|
|
744
|
+
# @api private
|
|
745
|
+
def drain_pipe
|
|
746
|
+
while exception.nil?
|
|
747
|
+
remaining_time = time_remaining_until_close_deadline
|
|
748
|
+
|
|
749
|
+
data = pipe_reader.read_nonblock(chunk_size, exception: false)
|
|
750
|
+
|
|
751
|
+
break if data.nil? # EOF: every copy of the pipe's write fd is closed
|
|
752
|
+
|
|
753
|
+
if remaining_time&.zero?
|
|
754
|
+
@truncated = true
|
|
755
|
+
break
|
|
756
|
+
end
|
|
757
|
+
|
|
758
|
+
data == :wait_readable ? pipe_reader.wait_readable(remaining_time) : write_data(data)
|
|
759
|
+
end
|
|
760
|
+
end
|
|
761
|
+
|
|
762
|
+
# Close both ends of the internal wake pipe
|
|
763
|
+
#
|
|
764
|
+
# Called from {#monitor}'s `ensure` block after `@state = :closed` is
|
|
765
|
+
# published, on every teardown path (a normal {#close}, a destination
|
|
766
|
+
# error, or an exception that ends the monitor loop). The `closed?` guards
|
|
767
|
+
# make it safe when a test helper has already closed the fds after killing
|
|
768
|
+
# the monitoring thread.
|
|
769
|
+
#
|
|
770
|
+
# @return [void]
|
|
771
|
+
# @api private
|
|
772
|
+
def close_wake_pipe
|
|
773
|
+
wake_writer.close unless wake_writer.closed?
|
|
774
|
+
wake_reader.close unless wake_reader.closed?
|
|
775
|
+
end
|
|
776
|
+
|
|
777
|
+
# The seconds left before the deadline set by {#close}
|
|
778
|
+
#
|
|
779
|
+
# @return [Numeric, nil] `nil` when no deadline is set (wait without a
|
|
780
|
+
# time limit), 0 when the deadline has passed
|
|
781
|
+
# @api private
|
|
782
|
+
def time_remaining_until_close_deadline
|
|
783
|
+
return nil if @close_deadline.nil?
|
|
784
|
+
|
|
785
|
+
[@close_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
|
|
786
|
+
end
|
|
449
787
|
end
|
|
450
788
|
end
|
|
@@ -62,10 +62,8 @@ module ProcessExecuter
|
|
|
62
62
|
#
|
|
63
63
|
def initialize(**options_hash)
|
|
64
64
|
@options_hash = allowed_options.transform_values(&:default).merge(options_hash)
|
|
65
|
-
@errors = []
|
|
66
|
-
assert_no_unknown_options
|
|
67
65
|
define_accessor_methods
|
|
68
|
-
|
|
66
|
+
validate!
|
|
69
67
|
end
|
|
70
68
|
|
|
71
69
|
# All the allowed options as a hash whose keys are the option names
|
|
@@ -151,20 +149,43 @@ module ProcessExecuter
|
|
|
151
149
|
#
|
|
152
150
|
# Subsequent hashes' values overwrite earlier ones for the same key.
|
|
153
151
|
#
|
|
152
|
+
# The merged options are checked the same way the constructor checks its
|
|
153
|
+
# options: unknown options and invalid option values raise a
|
|
154
|
+
# `ProcessExecuter::ArgumentError`. In that case, the current options
|
|
155
|
+
# object is left unchanged.
|
|
156
|
+
#
|
|
154
157
|
# @example
|
|
155
158
|
# options = MyOptions.new(option1: 'value1', option2: 'value2')
|
|
156
159
|
# h1 = { option2: 'new_value2' }
|
|
157
160
|
# h2 = { option3: 'value3' }
|
|
158
|
-
# options.merge!(h1, h2) => {option1: "value1", option2: "new_value2", option3: "value3"}
|
|
161
|
+
# options.merge!(h1, h2) # => options with {option1: "value1", option2: "new_value2", option3: "value3"}
|
|
162
|
+
#
|
|
163
|
+
# @example with an invalid option value
|
|
164
|
+
# options = MyOptions.new(option1: 'value1')
|
|
165
|
+
# begin
|
|
166
|
+
# options.merge!(option1: 1)
|
|
167
|
+
# rescue ProcessExecuter::ArgumentError => e
|
|
168
|
+
# e.message #=> "option1 must be a String but was 1"
|
|
169
|
+
# options.option1 #=> 'value1'
|
|
170
|
+
# end
|
|
159
171
|
#
|
|
160
|
-
# @param other_options_hashes [Array<Hash>] zero
|
|
172
|
+
# @param other_options_hashes [Array<Hash>] zero or more hashes to merge into the current options
|
|
161
173
|
#
|
|
162
174
|
# @return [self] the current options object with the merged options
|
|
163
175
|
#
|
|
176
|
+
# @raise [ProcessExecuter::ArgumentError] if the merged options contain an
|
|
177
|
+
# unknown option or an invalid option value
|
|
178
|
+
#
|
|
164
179
|
# @api public
|
|
165
180
|
#
|
|
166
181
|
def merge!(*other_options_hashes)
|
|
167
|
-
options_hash
|
|
182
|
+
original_options_hash = @options_hash
|
|
183
|
+
@options_hash = original_options_hash.dup.merge!(*other_options_hashes)
|
|
184
|
+
validate!
|
|
185
|
+
self
|
|
186
|
+
rescue ProcessExecuter::ArgumentError
|
|
187
|
+
@options_hash = original_options_hash
|
|
188
|
+
raise
|
|
168
189
|
end
|
|
169
190
|
|
|
170
191
|
# Returns a new options object formed by merging self with each of other_hashes
|
|
@@ -181,6 +202,9 @@ module ProcessExecuter
|
|
|
181
202
|
#
|
|
182
203
|
# @return [self.class]
|
|
183
204
|
#
|
|
205
|
+
# @raise [ProcessExecuter::ArgumentError] if the merged options contain an
|
|
206
|
+
# unknown option or an invalid option value
|
|
207
|
+
#
|
|
184
208
|
def merge(*other_options_hashes)
|
|
185
209
|
merged_options = other_options_hashes.reduce(options_hash, :merge)
|
|
186
210
|
self.class.new(**merged_options)
|
|
@@ -233,6 +257,23 @@ module ProcessExecuter
|
|
|
233
257
|
#
|
|
234
258
|
attr_reader :options_hash
|
|
235
259
|
|
|
260
|
+
# Run the full validation sequence over the current options hash
|
|
261
|
+
#
|
|
262
|
+
# Checks for unknown options and then validates option values, leaving
|
|
263
|
+
# `errors` empty on exit whether validation passes or raises.
|
|
264
|
+
#
|
|
265
|
+
# @return [void]
|
|
266
|
+
# @raise [ProcessExecuter::ArgumentError] if an unknown option or an
|
|
267
|
+
# invalid option value is found
|
|
268
|
+
# @api private
|
|
269
|
+
def validate!
|
|
270
|
+
@errors = []
|
|
271
|
+
assert_no_unknown_options
|
|
272
|
+
validate_options
|
|
273
|
+
ensure
|
|
274
|
+
@errors = []
|
|
275
|
+
end
|
|
276
|
+
|
|
236
277
|
# Raise an argument error for invalid option values
|
|
237
278
|
# @return [void]
|
|
238
279
|
# @raise [ProcessExecuter::ArgumentError] if any invalid option values are found
|
|
@@ -267,7 +308,7 @@ module ProcessExecuter
|
|
|
267
308
|
return if unknown_options.empty?
|
|
268
309
|
|
|
269
310
|
raise(
|
|
270
|
-
ArgumentError,
|
|
311
|
+
ProcessExecuter::ArgumentError,
|
|
271
312
|
"Unknown option#{'s' if unknown_options.count > 1}: #{unknown_options.join(', ')}"
|
|
272
313
|
)
|
|
273
314
|
end
|