polyphony 0.79 → 0.80

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/Gemfile.lock +2 -1
  4. data/examples/core/zlib_stream.rb +15 -0
  5. data/ext/polyphony/backend_common.c +2 -1
  6. data/ext/polyphony/backend_common.h +7 -2
  7. data/lib/polyphony/adapters/fs.rb +4 -0
  8. data/lib/polyphony/adapters/process.rb +14 -1
  9. data/lib/polyphony/adapters/redis.rb +28 -0
  10. data/lib/polyphony/adapters/sequel.rb +19 -1
  11. data/lib/polyphony/core/debug.rb +129 -72
  12. data/lib/polyphony/core/exceptions.rb +21 -6
  13. data/lib/polyphony/core/global_api.rb +228 -73
  14. data/lib/polyphony/core/resource_pool.rb +65 -20
  15. data/lib/polyphony/core/sync.rb +57 -12
  16. data/lib/polyphony/core/thread_pool.rb +42 -5
  17. data/lib/polyphony/core/throttler.rb +21 -5
  18. data/lib/polyphony/core/timer.rb +125 -1
  19. data/lib/polyphony/extensions/exception.rb +36 -6
  20. data/lib/polyphony/extensions/fiber.rb +238 -57
  21. data/lib/polyphony/extensions/io.rb +4 -2
  22. data/lib/polyphony/extensions/kernel.rb +9 -4
  23. data/lib/polyphony/extensions/object.rb +8 -0
  24. data/lib/polyphony/extensions/openssl.rb +3 -1
  25. data/lib/polyphony/extensions/socket.rb +458 -39
  26. data/lib/polyphony/extensions/thread.rb +108 -43
  27. data/lib/polyphony/extensions/timeout.rb +12 -1
  28. data/lib/polyphony/extensions.rb +1 -0
  29. data/lib/polyphony/net.rb +59 -0
  30. data/lib/polyphony/version.rb +1 -1
  31. data/lib/polyphony.rb +0 -2
  32. data/test/test_backend.rb +6 -2
  33. data/test/test_global_api.rb +0 -23
  34. data/test/test_resource_pool.rb +1 -1
  35. data/test/test_throttler.rb +0 -6
  36. data/test/test_trace.rb +87 -0
  37. metadata +9 -8
  38. data/lib/polyphony/core/channel.rb +0 -15
@@ -3,15 +3,25 @@
3
3
  require 'etc'
4
4
 
5
5
  module Polyphony
6
+
6
7
  # Implements a pool of threads
7
8
  class ThreadPool
9
+
10
+ # The pool size.
8
11
  attr_reader :size
9
12
 
13
+ # Runs the given block on an available thread from the default thread pool.
14
+ #
15
+ # @param &block [Proc] given block
16
+ # @return [any] return value of given block
10
17
  def self.process(&block)
11
18
  @default_pool ||= new
12
19
  @default_pool.process(&block)
13
20
  end
14
21
 
22
+ # Resets the default thread pool.
23
+ #
24
+ # @return [void]
15
25
  def self.reset
16
26
  return unless @default_pool
17
27
 
@@ -19,12 +29,20 @@ module Polyphony
19
29
  @default_pool = nil
20
30
  end
21
31
 
32
+ # Initializes the thread pool. The pool size defaults to the number of
33
+ # available CPU cores.
34
+ #
35
+ # @param size [Integer] number of threads in pool
22
36
  def initialize(size = Etc.nprocessors)
23
37
  @size = size
24
38
  @task_queue = Polyphony::Queue.new
25
39
  @threads = (1..@size).map { Thread.new { thread_loop } }
26
40
  end
27
41
 
42
+ # Runs the given block on an available thread from the pool.
43
+ #
44
+ # @param &block [Proc] given block
45
+ # @return [any] return value of block
28
46
  def process(&block)
29
47
  setup unless @task_queue
30
48
 
@@ -33,6 +51,12 @@ module Polyphony
33
51
  watcher.await
34
52
  end
35
53
 
54
+ # Adds a task to be performed asynchronously on a thread from the pool. This
55
+ # method does not block. The task will be performed once a thread becomes
56
+ # available.
57
+ #
58
+ # @param &block [Proc] given block
59
+ # @return [Polyphony::ThreadPool] self
36
60
  def cast(&block)
37
61
  setup unless @task_queue
38
62
 
@@ -40,16 +64,34 @@ module Polyphony
40
64
  self
41
65
  end
42
66
 
67
+ # Returns true if there are any currently running tasks, or any pending
68
+ # tasks waiting for a thread to become available.
69
+ #
70
+ # @return [bool] true if the pool is busy
43
71
  def busy?
44
72
  !@task_queue.empty?
45
73
  end
46
74
 
75
+ # Stops and waits for all threads in the queue to terminate.
76
+ def stop
77
+ @threads.each(&:kill)
78
+ @threads.each(&:join)
79
+ end
80
+
81
+ private
82
+
83
+ # Runs a processing loop on a worker thread.
84
+ #
85
+ # @return [void]
47
86
  def thread_loop
48
87
  while true
49
88
  run_queued_task
50
89
  end
51
90
  end
52
91
 
92
+ # Runs the first queued task in the task queue.
93
+ #
94
+ # @return [void]
53
95
  def run_queued_task
54
96
  (block, watcher) = @task_queue.shift
55
97
  result = block.()
@@ -57,10 +99,5 @@ module Polyphony
57
99
  rescue Exception => e
58
100
  watcher ? watcher.signal(e) : raise(e)
59
101
  end
60
-
61
- def stop
62
- @threads.each(&:kill)
63
- @threads.each(&:join)
64
- end
65
102
  end
66
103
  end
@@ -3,31 +3,47 @@
3
3
  module Polyphony
4
4
  # Implements general-purpose throttling
5
5
  class Throttler
6
+
7
+ # Initializes a throttler instance with the given rate.
8
+ #
9
+ # @param rate [Number] throttler rate in times per second
6
10
  def initialize(rate)
7
11
  @rate = rate_from_argument(rate)
8
12
  @min_dt = 1.0 / @rate
9
13
  @next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
10
14
  end
11
15
 
16
+ # call-seq:
17
+ # throttler.call { ... }
18
+ # throttler.process { ... }
19
+ #
20
+ # Invokes the throttler with the given block. The throttler will
21
+ # automatically introduce a delay to keep to the maximum specified rate.
22
+ # The throttler instance is passed to the given block.
23
+ #
24
+ # @return [any] given block's return value
12
25
  def call
13
26
  now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
14
27
  delta = @next_time - now
15
28
  Polyphony.backend_sleep(delta) if delta > 0
16
- yield self
29
+ result = yield self
17
30
 
18
31
  while true
19
32
  @next_time += @min_dt
20
33
  break if @next_time > now
21
34
  end
35
+
36
+ result
22
37
  end
23
38
  alias_method :process, :call
24
39
 
25
- def stop
26
- @stop = true
27
- end
28
-
29
40
  private
30
41
 
42
+ # Converts the given argument to a rate. If a hash is given, the throttler's
43
+ # rate is computed from the value of either the `:interval` or `:rate` keys.
44
+ #
45
+ # @param arg [Number, Hash] rate argument
46
+ # @return [Number] rate in times per second
31
47
  def rate_from_argument(arg)
32
48
  return arg if arg.is_a?(Numeric)
33
49
 
@@ -1,17 +1,34 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Polyphony
4
- # Implements a common timer for running multiple timeouts
4
+
5
+ # Implements a common timer for running multiple timeouts. This class may be
6
+ # used to reduce the timer granularity in case a large number of timeouts is
7
+ # used concurrently. This class basically provides the same methods as global
8
+ # methods concerned with timeouts, such as `#cancel_after`, `#every` etc.
5
9
  class Timer
10
+
11
+ # Initializes a new timer with the given resolution.
12
+ #
13
+ # @param tag [any] tag to use for the timer's fiber
14
+ # @param resolution: [Number] timer granularity in seconds or fractions thereof
6
15
  def initialize(tag = nil, resolution:)
7
16
  @fiber = spin_loop(tag, interval: resolution) { update }
8
17
  @timeouts = {}
9
18
  end
10
19
 
20
+ # Stops the timer's associated fiber.
21
+ #
22
+ # @return [Polyphony::Timer] self
11
23
  def stop
12
24
  @fiber.stop
25
+ self
13
26
  end
14
27
 
28
+ # Sleeps for the given duration.
29
+ #
30
+ # @param duration [Number] sleep duration in seconds
31
+ # @return [void]
15
32
  def sleep(duration)
16
33
  fiber = Fiber.current
17
34
  @timeouts[fiber] = {
@@ -23,6 +40,12 @@ module Polyphony
23
40
  @timeouts.delete(fiber)
24
41
  end
25
42
 
43
+ # Spins up a fiber that will run the given block after sleeping for the
44
+ # given delay.
45
+ #
46
+ # @param delay [Number] delay in seconds before running the given block
47
+ # @param &block [Proc] block to run
48
+ # @return [Fiber] spun fiber
26
49
  def after(interval, &block)
27
50
  spin do
28
51
  self.sleep interval
@@ -30,6 +53,12 @@ module Polyphony
30
53
  end
31
54
  end
32
55
 
56
+ # Runs the given block in an infinite loop with a regular interval between
57
+ # consecutive iterations.
58
+ #
59
+ # @param interval [Number] interval between consecutive iterations in seconds
60
+ # @param &block [Proc] block to run
61
+ # @return [void]
33
62
  def every(interval)
34
63
  fiber = Fiber.current
35
64
  @timeouts[fiber] = {
@@ -45,6 +74,44 @@ module Polyphony
45
74
  @timeouts.delete(fiber)
46
75
  end
47
76
 
77
+ # call-seq:
78
+ # timer.cancel_after(interval) { ... }
79
+ # timer.cancel_after(interval, with_exception: exception) { ... }
80
+ # timer.cancel_after(interval, with_exception: [klass, message]) { ... }
81
+ # timer.cancel_after(interval) { |timeout| ... }
82
+ # timer.cancel_after(interval, with_exception: exception) { |timeout| ... }
83
+ # timer.cancel_after(interval, with_exception: [klass, message]) { |timeout| ... }
84
+ #
85
+ # Runs the given block after setting up a cancellation timer for
86
+ # cancellation. If the cancellation timer elapses, the execution will be
87
+ # interrupted with an exception defaulting to `Polyphony::Cancel`.
88
+ #
89
+ # This method should be used when a timeout should cause an exception to be
90
+ # propagated down the call stack or up the fiber tree.
91
+ #
92
+ # Example of normal use:
93
+ #
94
+ # def read_from_io_with_timeout(io)
95
+ # timer.cancel_after(10) { io.read }
96
+ # rescue Polyphony::Cancel
97
+ # nil
98
+ # end
99
+ #
100
+ # The timeout period can be reset using `Timer#reset`, as shown in the
101
+ # following example:
102
+ #
103
+ # timer.cancel_after(10) do
104
+ # loop do
105
+ # msg = socket.gets
106
+ # timer.reset
107
+ # handle_msg(msg)
108
+ # end
109
+ # end
110
+ #
111
+ # @param interval [Number] timout in seconds
112
+ # @param with_exception: [Class, Exception] exception or exception class
113
+ # @param &block [Proc] block to execute
114
+ # @return [any] block's return value
48
115
  def cancel_after(interval, with_exception: Polyphony::Cancel)
49
116
  fiber = Fiber.current
50
117
  @timeouts[fiber] = {
@@ -57,6 +124,48 @@ module Polyphony
57
124
  @timeouts.delete(fiber)
58
125
  end
59
126
 
127
+ # call-seq:
128
+ # timer.move_on_after(interval) { ... }
129
+ # timer.move_on_after(interval, with_value: value) { ... }
130
+ # timer.move_on_after(interval) { |canceller| ... }
131
+ # timer.move_on_after(interval, with_value: value) { |canceller| ... }
132
+ #
133
+ # Runs the given block after setting up a cancellation timer for
134
+ # cancellation. If the cancellation timer elapses, the execution will be
135
+ # interrupted with a `Polyphony::MoveOn` exception, which will be rescued,
136
+ # and with cause the operation to return the given value.
137
+ #
138
+ # This method should be used when a timeout is to be handled locally,
139
+ # without generating an exception that is to propagated down the call stack
140
+ # or up the fiber tree.
141
+ #
142
+ # Example of normal use:
143
+ #
144
+ # timer.move_on_after(10) {
145
+ # sleep 60
146
+ # 42
147
+ # } #=> nil
148
+ #
149
+ # timer.move_on_after(10, with_value: :oops) {
150
+ # sleep 60
151
+ # 42
152
+ # } #=> :oops
153
+ #
154
+ # The timeout period can be reset using `Timer#reset`, as shown in the
155
+ # following example:
156
+ #
157
+ # timer.move_on_after(10) do
158
+ # loop do
159
+ # msg = socket.gets
160
+ # timer.reset
161
+ # handle_msg(msg)
162
+ # end
163
+ # end
164
+ #
165
+ # @param interval [Number] timout in seconds
166
+ # @param with_value: [any] return value in case of timeout
167
+ # @param &block [Proc] block to execute
168
+ # @return [any] block's return value
60
169
  def move_on_after(interval, with_value: nil)
61
170
  fiber = Fiber.current
62
171
  @timeouts[fiber] = {
@@ -71,6 +180,9 @@ module Polyphony
71
180
  @timeouts.delete(fiber)
72
181
  end
73
182
 
183
+ # Resets the timeout for the current fiber.
184
+ #
185
+ # @return [void]
74
186
  def reset
75
187
  record = @timeouts[Fiber.current]
76
188
  return unless record
@@ -80,21 +192,33 @@ module Polyphony
80
192
 
81
193
  private
82
194
 
195
+ # Returns the current monotonic clock value.
196
+ #
197
+ # @return [Number] monotonic clock value in seconds
83
198
  def now
84
199
  ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
85
200
  end
86
201
 
202
+ # Converts a timeout record's exception spec to an exception instance.
203
+ #
204
+ # @param record [Array, Class, Exception, String] exception spec
205
+ # @return [Exception] exception instance
87
206
  def timeout_exception(record)
88
207
  case (exception = record[:exception])
89
208
  when Array
90
209
  exception[0].new(exception[1])
91
210
  when Class
92
211
  exception.new
212
+ when Exception
213
+ exception
93
214
  else
94
215
  RuntimeError.new(exception)
95
216
  end
96
217
  end
97
218
 
219
+ # Runs a timer iteration, invoking any timeouts that are due.
220
+ #
221
+ # @return [void]
98
222
  def update
99
223
  return if @timeouts.empty?
100
224
 
@@ -1,20 +1,38 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Exeption overrides
3
+ # Extensions to the Exception class
4
4
  class ::Exception
5
5
  class << self
6
+
7
+ # Set to true to disable sanitizing the backtrace (to remove frames occuring
8
+ # in the Polyphony code itself.)
6
9
  attr_accessor :__disable_sanitized_backtrace__
7
10
  end
8
11
 
9
- attr_accessor :source_fiber, :raising_fiber
12
+ # Set to the fiber in which the exception was *originally* raised (in case the
13
+ # exception was not caught.) The exception will propagate up the fiber tree,
14
+ # allowing it to be caught in any of the fiber's ancestors, while the
15
+ # `@source_fiber`` attribute will continue pointing to the original fiber.
16
+ attr_accessor :source_fiber
17
+
18
+ # Set to the fiber from which the exception was raised.
19
+ attr_accessor :raising_fiber
10
20
 
11
21
  alias_method :orig_initialize, :initialize
22
+
23
+ # Initializes the exception with the given arguments.
12
24
  def initialize(*args)
13
25
  @raising_fiber = Fiber.current
14
26
  orig_initialize(*args)
15
27
  end
16
28
 
17
29
  alias_method :orig_backtrace, :backtrace
30
+
31
+ # Returns the backtrace for the exception. If
32
+ # `Exception.__disable_sanitized_backtrace__` is not true, any stack frames
33
+ # occuring in Polyphony's code will be removed from the backtrace.
34
+ #
35
+ # @return [Array<String>] backtrace
18
36
  def backtrace
19
37
  unless @backtrace_called
20
38
  @backtrace_called = true
@@ -24,6 +42,17 @@ class ::Exception
24
42
  sanitized_backtrace
25
43
  end
26
44
 
45
+ # Raises the exception. this method is a simple wrapper to `Kernel.raise`. It
46
+ # is overriden in the `Polyphony::Interjection` exception class.
47
+ def invoke
48
+ Kernel.raise(self)
49
+ end
50
+
51
+ private
52
+
53
+ # Returns a sanitized backtrace for the exception.
54
+ #
55
+ # @return [Array<String>] sanitized backtrace
27
56
  def sanitized_backtrace
28
57
  return sanitize(orig_backtrace) unless @raising_fiber
29
58
 
@@ -33,13 +62,14 @@ class ::Exception
33
62
 
34
63
  POLYPHONY_DIR = File.expand_path(File.join(__dir__, '..'))
35
64
 
65
+ # Sanitizes the backtrace by removing any frames occuring in Polyphony's code
66
+ # base.
67
+ #
68
+ # @param backtrace [Array<String>] unsanitized backtrace
69
+ # @return [Array<String>] sanitized backtrace
36
70
  def sanitize(backtrace)
37
71
  return backtrace if ::Exception.__disable_sanitized_backtrace__
38
72
 
39
73
  backtrace.reject { |l| l[POLYPHONY_DIR] }
40
74
  end
41
-
42
- def invoke
43
- Kernel.raise(self)
44
- end
45
75
  end