async-container 0.16.2 → 0.16.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/async/container.rb +0 -1
- data/lib/async/container/best.rb +9 -3
- data/lib/async/container/channel.rb +13 -0
- data/lib/async/container/controller.rb +36 -17
- data/lib/async/container/error.rb +13 -0
- data/lib/async/container/forked.rb +5 -1
- data/lib/async/container/generic.rb +51 -13
- data/lib/async/container/group.rb +22 -14
- data/lib/async/container/hybrid.rb +16 -2
- data/lib/async/container/keyed.rb +12 -0
- data/lib/async/container/notify.rb +4 -5
- data/lib/async/container/notify/client.rb +16 -1
- data/lib/async/container/notify/console.rb +9 -22
- data/lib/async/container/notify/pipe.rb +11 -24
- data/lib/async/container/notify/server.rb +1 -2
- data/lib/async/container/notify/socket.rb +14 -1
- data/lib/async/container/process.rb +26 -0
- data/lib/async/container/statistics.rb +16 -0
- data/lib/async/container/thread.rb +42 -4
- data/lib/async/container/threaded.rb +5 -1
- data/lib/async/container/version.rb +1 -1
- metadata +31 -74
- data/.editorconfig +0 -6
- data/.github/workflows/development.yml +0 -36
- data/.gitignore +0 -21
- data/.rspec +0 -3
- data/.travis.yml +0 -21
- data/.yardopts +0 -1
- data/Gemfile +0 -19
- data/Guardfile +0 -14
- data/README.md +0 -140
- data/Rakefile +0 -8
- data/async-container.gemspec +0 -34
- data/examples/async.rb +0 -22
- data/examples/channel.rb +0 -45
- data/examples/channels/client.rb +0 -104
- data/examples/container.rb +0 -33
- data/examples/isolate.rb +0 -36
- data/examples/minimal.rb +0 -94
- data/examples/test.rb +0 -51
- data/examples/threads.rb +0 -25
- data/examples/title.rb +0 -13
- data/examples/udppipe.rb +0 -35
- data/spec/async/container/controller_spec.rb +0 -106
- data/spec/async/container/forked_spec.rb +0 -61
- data/spec/async/container/hybrid_spec.rb +0 -36
- data/spec/async/container/notify/notify.rb +0 -19
- data/spec/async/container/notify/pipe_spec.rb +0 -48
- data/spec/async/container/notify_spec.rb +0 -56
- data/spec/async/container/shared_examples.rb +0 -80
- data/spec/async/container/threaded_spec.rb +0 -35
- data/spec/async/container_spec.rb +0 -41
- data/spec/spec_helper.rb +0 -15
@@ -21,7 +21,6 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
require 'fiber'
|
24
|
-
|
25
24
|
require 'async/clock'
|
26
25
|
|
27
26
|
require_relative 'error'
|
@@ -30,6 +29,7 @@ module Async
|
|
30
29
|
module Container
|
31
30
|
# Manages a group of running processes.
|
32
31
|
class Group
|
32
|
+
# Initialize an empty group.
|
33
33
|
def initialize
|
34
34
|
@running = {}
|
35
35
|
|
@@ -40,19 +40,25 @@ module Async
|
|
40
40
|
# @attribute [Hash<IO, Fiber>] the running tasks, indexed by IO.
|
41
41
|
attr :running
|
42
42
|
|
43
|
+
# Whether the group contains any running processes.
|
44
|
+
# @returns [Boolean]
|
43
45
|
def running?
|
44
46
|
@running.any?
|
45
47
|
end
|
46
48
|
|
49
|
+
# Whether the group contains any running processes.
|
50
|
+
# @returns [Boolean]
|
47
51
|
def any?
|
48
52
|
@running.any?
|
49
53
|
end
|
50
54
|
|
55
|
+
# Whether the group is empty.
|
56
|
+
# @returns [Boolean]
|
51
57
|
def empty?
|
52
58
|
@running.empty?
|
53
59
|
end
|
54
60
|
|
55
|
-
#
|
61
|
+
# Sleep for at most the specified duration until some state change occurs.
|
56
62
|
def sleep(duration)
|
57
63
|
self.resume
|
58
64
|
self.suspend
|
@@ -60,6 +66,7 @@ module Async
|
|
60
66
|
self.wait_for_children(duration)
|
61
67
|
end
|
62
68
|
|
69
|
+
# Begin any outstanding queued processes and wait for them indefinitely.
|
63
70
|
def wait
|
64
71
|
self.resume
|
65
72
|
|
@@ -68,26 +75,33 @@ module Async
|
|
68
75
|
end
|
69
76
|
end
|
70
77
|
|
78
|
+
# Interrupt all running processes.
|
79
|
+
# This resumes the controlling fiber with an instance of {Interrupt}.
|
71
80
|
def interrupt
|
81
|
+
Async.logger.debug(self, "Sending interrupt to #{@running.size} running processes...")
|
72
82
|
@running.each_value do |fiber|
|
73
83
|
fiber.resume(Interrupt)
|
74
84
|
end
|
75
85
|
end
|
76
86
|
|
87
|
+
# Terminate all running processes.
|
88
|
+
# This resumes the controlling fiber with an instance of {Terminate}.
|
77
89
|
def terminate
|
90
|
+
Async.logger.debug(self, "Sending terminate to #{@running.size} running processes...")
|
78
91
|
@running.each_value do |fiber|
|
79
92
|
fiber.resume(Terminate)
|
80
93
|
end
|
81
94
|
end
|
82
95
|
|
96
|
+
# Stop all child processes using {#terminate}.
|
97
|
+
# @parameter timeout [Boolean | Numeric | Nil] If specified, invoke a graceful shutdown using {#interrupt} first.
|
83
98
|
def stop(timeout = 1)
|
84
|
-
#
|
99
|
+
# Use a default timeout if not specified:
|
100
|
+
timeout = 1 if timeout == true
|
101
|
+
|
85
102
|
if timeout
|
86
103
|
start_time = Async::Clock.now
|
87
104
|
|
88
|
-
# Use a default timeout if not specified:
|
89
|
-
timeout = 1 if timeout == true
|
90
|
-
|
91
105
|
self.interrupt
|
92
106
|
|
93
107
|
while self.any?
|
@@ -103,14 +117,6 @@ module Async
|
|
103
117
|
end
|
104
118
|
end
|
105
119
|
|
106
|
-
# Timeout can also be `graceful = false`:
|
107
|
-
if timeout
|
108
|
-
self.interrupt
|
109
|
-
self.sleep(timeout)
|
110
|
-
end
|
111
|
-
|
112
|
-
self.wait_for_children(duration)
|
113
|
-
|
114
120
|
# Terminate all children:
|
115
121
|
self.terminate
|
116
122
|
|
@@ -118,6 +124,7 @@ module Async
|
|
118
124
|
self.wait
|
119
125
|
end
|
120
126
|
|
127
|
+
# Wait for a message in the specified {Channel}.
|
121
128
|
def wait_for(channel)
|
122
129
|
io = channel.in
|
123
130
|
|
@@ -144,6 +151,7 @@ module Async
|
|
144
151
|
|
145
152
|
def wait_for_children(duration = nil)
|
146
153
|
if !@running.empty?
|
154
|
+
# Maybe consider using a proper event loop here:
|
147
155
|
readable, _, _ = ::IO.select(@running.keys, nil, nil, duration)
|
148
156
|
|
149
157
|
readable&.each do |io|
|
@@ -25,7 +25,12 @@ require_relative 'threaded'
|
|
25
25
|
|
26
26
|
module Async
|
27
27
|
module Container
|
28
|
+
# Provides a hybrid multi-process multi-thread container.
|
28
29
|
class Hybrid < Forked
|
30
|
+
# Run multiple instances of the same block in the container.
|
31
|
+
# @parameter count [Integer] The number of instances to start.
|
32
|
+
# @parameter forks [Integer] The number of processes to fork.
|
33
|
+
# @parameter threads [Integer] the number of threads to start.
|
29
34
|
def run(count: nil, forks: nil, threads: nil, **options, &block)
|
30
35
|
processor_count = Container.processor_count
|
31
36
|
count ||= processor_count ** 2
|
@@ -33,12 +38,21 @@ module Async
|
|
33
38
|
threads = (count / forks).ceil
|
34
39
|
|
35
40
|
forks.times do
|
36
|
-
self.spawn(**options) do
|
37
|
-
container = Threaded
|
41
|
+
self.spawn(**options) do |instance|
|
42
|
+
container = Threaded.new
|
38
43
|
|
39
44
|
container.run(count: threads, **options, &block)
|
40
45
|
|
46
|
+
container.wait_until_ready
|
47
|
+
instance.ready!
|
48
|
+
|
41
49
|
container.wait
|
50
|
+
rescue Async::Container::Terminate
|
51
|
+
# Stop it immediately:
|
52
|
+
container.stop(false)
|
53
|
+
ensure
|
54
|
+
# Stop it gracefully (also code path for Interrupt):
|
55
|
+
container.stop
|
42
56
|
end
|
43
57
|
end
|
44
58
|
|
@@ -22,6 +22,8 @@
|
|
22
22
|
|
23
23
|
module Async
|
24
24
|
module Container
|
25
|
+
# Tracks a key/value pair such that unmarked keys can be identified and cleaned up.
|
26
|
+
# This helps implement persistent processes that start up child processes per directory or configuration file. If those directories and/or configuration files are removed, the child process can then be cleaned up automatically, because those key/value pairs will not be marked when reloading the container.
|
25
27
|
class Keyed
|
26
28
|
def initialize(key, value)
|
27
29
|
@key = key
|
@@ -29,21 +31,31 @@ module Async
|
|
29
31
|
@marked = true
|
30
32
|
end
|
31
33
|
|
34
|
+
# The key. Normally a symbol or a file-system path.
|
35
|
+
# @attribute [Object]
|
32
36
|
attr :key
|
37
|
+
|
38
|
+
# The value. Normally a child instance of some sort.
|
39
|
+
# @attribute [Object]
|
33
40
|
attr :value
|
34
41
|
|
42
|
+
# Has the instance been marked?
|
43
|
+
# @returns [Boolean]
|
35
44
|
def marked?
|
36
45
|
@marked
|
37
46
|
end
|
38
47
|
|
48
|
+
# Mark the instance. This will indiciate that the value is still in use/active.
|
39
49
|
def mark!
|
40
50
|
@marked = true
|
41
51
|
end
|
42
52
|
|
53
|
+
# Clear the instance. This is normally done before reloading a container.
|
43
54
|
def clear!
|
44
55
|
@marked = false
|
45
56
|
end
|
46
57
|
|
58
|
+
# Stop the instance if it was not marked.
|
47
59
|
def stop?
|
48
60
|
unless @marked
|
49
61
|
@value.stop
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -28,12 +27,12 @@ require_relative 'notify/console'
|
|
28
27
|
module Async
|
29
28
|
module Container
|
30
29
|
module Notify
|
31
|
-
|
32
|
-
@@client = nil
|
30
|
+
@client = nil
|
33
31
|
|
32
|
+
# Select the best available notification client.
|
33
|
+
# We cache the client on a per-process basis. Because that's the relevant scope for process readiness protocols.
|
34
34
|
def self.open!
|
35
|
-
|
36
|
-
@@client ||= (
|
35
|
+
@client ||= (
|
37
36
|
Pipe.open! ||
|
38
37
|
Socket.open! ||
|
39
38
|
Console.open!
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -23,12 +22,17 @@
|
|
23
22
|
|
24
23
|
module Async
|
25
24
|
module Container
|
25
|
+
# Handles the details of several process readiness protocols.
|
26
26
|
module Notify
|
27
27
|
class Client
|
28
|
+
# Notify the parent controller that the child has become ready, with a brief status message.
|
29
|
+
# @parameters message [Hash] Additional details to send with the message.
|
28
30
|
def ready!(**message)
|
29
31
|
send(ready: true, **message)
|
30
32
|
end
|
31
33
|
|
34
|
+
# Notify the parent controller that the child is reloading.
|
35
|
+
# @parameters message [Hash] Additional details to send with the message.
|
32
36
|
def reloading!(**message)
|
33
37
|
message[:ready] = false
|
34
38
|
message[:reloading] = true
|
@@ -37,6 +41,8 @@ module Async
|
|
37
41
|
send(**message)
|
38
42
|
end
|
39
43
|
|
44
|
+
# Notify the parent controller that the child is restarting.
|
45
|
+
# @parameters message [Hash] Additional details to send with the message.
|
40
46
|
def restarting!(**message)
|
41
47
|
message[:ready] = false
|
42
48
|
message[:reloading] = true
|
@@ -45,14 +51,23 @@ module Async
|
|
45
51
|
send(**message)
|
46
52
|
end
|
47
53
|
|
54
|
+
# Notify the parent controller that the child is stopping.
|
55
|
+
# @parameters message [Hash] Additional details to send with the message.
|
48
56
|
def stopping!(**message)
|
49
57
|
message[:stopping] = true
|
58
|
+
|
59
|
+
send(**message)
|
50
60
|
end
|
51
61
|
|
62
|
+
# Notify the parent controller of a status change.
|
63
|
+
# @parameters text [String] The details of the status change.
|
52
64
|
def status!(text)
|
53
65
|
send(status: text)
|
54
66
|
end
|
55
67
|
|
68
|
+
# Notify the parent controller of an error condition.
|
69
|
+
# @parameters text [String] The details of the error condition.
|
70
|
+
# @parameters message [Hash] Additional details to send with the message.
|
56
71
|
def error!(text, **message)
|
57
72
|
send(status: text, **message)
|
58
73
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -28,39 +27,27 @@ require 'console/logger'
|
|
28
27
|
module Async
|
29
28
|
module Container
|
30
29
|
module Notify
|
30
|
+
# Implements a general process readiness protocol with output to the local console.
|
31
31
|
class Console < Client
|
32
|
+
# Open a notification client attached to the current console.
|
32
33
|
def self.open!(logger = ::Console.logger)
|
33
34
|
self.new(logger)
|
34
35
|
end
|
35
36
|
|
37
|
+
# Initialize the notification client.
|
38
|
+
# @parameter logger [Console::Logger] The console logger instance to send messages to.
|
36
39
|
def initialize(logger)
|
37
40
|
@logger = logger
|
38
41
|
end
|
39
42
|
|
43
|
+
# Send a message to the console.
|
40
44
|
def send(level: :debug, **message)
|
41
|
-
@logger.send(level, self) {message
|
42
|
-
end
|
43
|
-
|
44
|
-
def ready!(**message)
|
45
|
-
send(ready: true, **message)
|
46
|
-
end
|
47
|
-
|
48
|
-
def restarting!(**message)
|
49
|
-
message[:ready] = false
|
50
|
-
message[:reloading] = true
|
51
|
-
message[:status] ||= "Restarting..."
|
52
|
-
|
53
|
-
send(**message)
|
54
|
-
end
|
55
|
-
|
56
|
-
def reloading!(**message)
|
57
|
-
message[:ready] = false
|
58
|
-
message[:reloading] = true
|
59
|
-
message[:status] ||= "Reloading..."
|
60
|
-
|
61
|
-
send(**message)
|
45
|
+
@logger.send(level, self) {message}
|
62
46
|
end
|
63
47
|
|
48
|
+
# Send an error message to the console.
|
49
|
+
# @parameters text [String] The details of the error condition.
|
50
|
+
# @parameters message [Hash] Additional details to send with the message.
|
64
51
|
def error!(text, **message)
|
65
52
|
send(status: text, level: :error, **message)
|
66
53
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -28,9 +27,12 @@ require 'json'
|
|
28
27
|
module Async
|
29
28
|
module Container
|
30
29
|
module Notify
|
30
|
+
# Implements a process readiness protocol using an inherited pipe file descriptor.
|
31
31
|
class Pipe < Client
|
32
|
+
# The environment variable key which contains the pipe file descriptor.
|
32
33
|
NOTIFY_PIPE = 'NOTIFY_PIPE'
|
33
34
|
|
35
|
+
# Open a notification client attached to the current {NOTIFY_PIPE} if possible.
|
34
36
|
def self.open!(environment = ENV)
|
35
37
|
if descriptor = environment.delete(NOTIFY_PIPE)
|
36
38
|
self.new(::IO.for_fd(descriptor.to_i))
|
@@ -41,6 +43,8 @@ module Async
|
|
41
43
|
return nil
|
42
44
|
end
|
43
45
|
|
46
|
+
# Initialize the notification client.
|
47
|
+
# @parameter io [IO] An IO instance used for sending messages.
|
44
48
|
def initialize(io)
|
45
49
|
@io = io
|
46
50
|
end
|
@@ -56,9 +60,10 @@ module Async
|
|
56
60
|
environment[NOTIFY_PIPE] = notify_pipe.to_s
|
57
61
|
|
58
62
|
# Use stdout if it's not redirected:
|
59
|
-
|
60
|
-
|
61
|
-
|
63
|
+
# This can cause issues if the user expects stdout to be connected to a terminal.
|
64
|
+
# elsif !options.key?(:out)
|
65
|
+
# options[:out] = @io
|
66
|
+
# environment[NOTIFY_PIPE] = "1"
|
62
67
|
|
63
68
|
# Use fileno 3 if it's available:
|
64
69
|
elsif !options.key?(3)
|
@@ -71,6 +76,8 @@ module Async
|
|
71
76
|
end
|
72
77
|
end
|
73
78
|
|
79
|
+
# Formats the message using JSON and sends it to the parent controller.
|
80
|
+
# This is suitable for use with {Channel}.
|
74
81
|
def send(**message)
|
75
82
|
data = ::JSON.dump(message)
|
76
83
|
|
@@ -78,26 +85,6 @@ module Async
|
|
78
85
|
@io.flush
|
79
86
|
end
|
80
87
|
|
81
|
-
def ready!(**message)
|
82
|
-
send(ready: true, **message)
|
83
|
-
end
|
84
|
-
|
85
|
-
def restarting!(**message)
|
86
|
-
message[:ready] = false
|
87
|
-
message[:reloading] = true
|
88
|
-
message[:status] ||= "Restarting..."
|
89
|
-
|
90
|
-
send(**message)
|
91
|
-
end
|
92
|
-
|
93
|
-
def reloading!(**message)
|
94
|
-
message[:ready] = false
|
95
|
-
message[:reloading] = true
|
96
|
-
message[:status] ||= "Reloading..."
|
97
|
-
|
98
|
-
send(**message)
|
99
|
-
end
|
100
|
-
|
101
88
|
private
|
102
89
|
|
103
90
|
def environment_for(arguments)
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -98,7 +97,7 @@ module Async
|
|
98
97
|
|
99
98
|
def receive
|
100
99
|
while true
|
101
|
-
data,
|
100
|
+
data, _address, _flags, *_controls = @bound.recvmsg(MAXIMUM_MESSAGE_SIZE)
|
102
101
|
|
103
102
|
message = Server.load(data)
|
104
103
|
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
#
|
4
3
|
# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
|
5
4
|
#
|
6
5
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
@@ -30,21 +29,31 @@ require 'kernel/sync'
|
|
30
29
|
module Async
|
31
30
|
module Container
|
32
31
|
module Notify
|
32
|
+
# Implements the systemd NOTIFY_SOCKET process readiness protocol.
|
33
|
+
# See <https://www.freedesktop.org/software/systemd/man/sd_notify.html> for more details of the underlying protocol.
|
33
34
|
class Socket < Client
|
35
|
+
# The name of the environment variable which contains the path to the notification socket.
|
34
36
|
NOTIFY_SOCKET = 'NOTIFY_SOCKET'
|
37
|
+
|
38
|
+
# The maximum allowed size of the UDP message.
|
35
39
|
MAXIMUM_MESSAGE_SIZE = 4096
|
36
40
|
|
41
|
+
# Open a notification client attached to the current {NOTIFY_SOCKET} if possible.
|
37
42
|
def self.open!(environment = ENV)
|
38
43
|
if path = environment.delete(NOTIFY_SOCKET)
|
39
44
|
self.new(path)
|
40
45
|
end
|
41
46
|
end
|
42
47
|
|
48
|
+
# Initialize the notification client.
|
49
|
+
# @parameter path [String] The path to the UNIX socket used for sending messages to the process manager.
|
43
50
|
def initialize(path)
|
44
51
|
@path = path
|
45
52
|
@endpoint = IO::Endpoint.unix(path, ::Socket::SOCK_DGRAM)
|
46
53
|
end
|
47
54
|
|
55
|
+
# Dump a message in the format requied by `sd_notify`.
|
56
|
+
# @parameter message [Hash] Keys and values should be string convertible objects. Values which are `true`/`false` are converted to `1`/`0` respectively.
|
48
57
|
def dump(message)
|
49
58
|
buffer = String.new
|
50
59
|
|
@@ -62,6 +71,8 @@ module Async
|
|
62
71
|
return buffer
|
63
72
|
end
|
64
73
|
|
74
|
+
# Send the given message.
|
75
|
+
# @parameter message [Hash]
|
65
76
|
def send(**message)
|
66
77
|
data = dump(message)
|
67
78
|
|
@@ -76,6 +87,8 @@ module Async
|
|
76
87
|
end
|
77
88
|
end
|
78
89
|
|
90
|
+
# Send the specified error.
|
91
|
+
# `sd_notify` requires an `errno` key, which defaults to `-1` to indicate a generic error.
|
79
92
|
def error!(text, **message)
|
80
93
|
message[:errno] ||= -1
|
81
94
|
|