opee 0.0.5 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +22 -6
- data/lib/opee.rb +2 -0
- data/lib/opee/actor.rb +83 -6
- data/lib/opee/collector.rb +103 -0
- data/lib/opee/env.rb +97 -1
- data/lib/opee/errors.rb +2 -0
- data/lib/opee/job.rb +43 -0
- data/lib/opee/log.rb +40 -0
- data/lib/opee/version.rb +1 -1
- data/lib/opee/workqueue.rb +31 -0
- data/test/tc_opee_collector.rb +152 -0
- data/test/ts_opee.rb +1 -0
- metadata +7 -3
data/README.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# Opee gem
|
2
|
-
An experimental Object-
|
2
|
+
An experimental Object-based Parallel Evaluation Environment
|
3
3
|
|
4
4
|
The purpose of this gem is to explore setting up an environment that will run
|
5
5
|
completely in parallel with minimum use of mutex synchronization. Actors only
|
@@ -14,17 +14,35 @@ This is no where close to being ready for prime time.
|
|
14
14
|
|
15
15
|
Any comments, thoughts, or suggestions are welcome.
|
16
16
|
|
17
|
+
## <a name="links">Links of Interest</a>
|
18
|
+
|
19
|
+
[Not Your Usual Threading](http://www.ohler.com/dev/http://www.ohler.com/dev/not_your_usual_threading/not_your_usual_threading.html) discussion about how OPEE use changes the way multi-threaded systems are designed.
|
20
|
+
|
17
21
|
## <a name="release">Release Notes</a>
|
18
22
|
|
19
|
-
### Release 0.0
|
23
|
+
### Release 0.1.0
|
24
|
+
|
25
|
+
- Documented classes.
|
20
26
|
|
21
|
-
- Added
|
27
|
+
- Added a rescuer attribute to the Env for global error handling.
|
22
28
|
|
23
|
-
- Added
|
29
|
+
- Added Collector and Job classes along with unit tests.
|
24
30
|
|
25
31
|
# Plans and Notes
|
26
32
|
|
33
|
+
- add name or label to Actors so they can be discovered by name or symbol
|
34
|
+
|
27
35
|
- pick a problem to test against
|
36
|
+
- drop file path into dir_wq
|
37
|
+
- pick up and decompose
|
38
|
+
- if dir then drop into queue again
|
39
|
+
- if file then send to filter actor
|
40
|
+
- may need to read start to determine text if no extention
|
41
|
+
- check for #! something and assume text/script
|
42
|
+
- if not one of know extension then search low chars that are not \n\r\t or other well knows in text
|
43
|
+
- cache text if file is read in
|
44
|
+
- if text send to through work queue density checker then to summary
|
45
|
+
|
28
46
|
- file density distribution
|
29
47
|
- find all files under a directory
|
30
48
|
- detemine if the file is a text or bin file and only keep text files
|
@@ -34,8 +52,6 @@ Any comments, thoughts, or suggestions are welcome.
|
|
34
52
|
- wait_finish
|
35
53
|
- ask to print or write report
|
36
54
|
|
37
|
-
- describe patterns for use
|
38
|
-
|
39
55
|
### License:
|
40
56
|
|
41
57
|
Copyright (c) 2012, Peter Ohler
|
data/lib/opee.rb
CHANGED
data/lib/opee/actor.rb
CHANGED
@@ -1,12 +1,29 @@
|
|
1
1
|
|
2
2
|
module Opee
|
3
|
-
|
3
|
+
# The Actor class is the base class for all the asynchronous Objects in
|
4
|
+
# OPEE. It accepts requests through the ask() method and excutes those
|
5
|
+
# methods on a thread dedicated to the Actor.
|
4
6
|
class Actor
|
7
|
+
# value of @state that indicates the Actor is not currently processing requests
|
5
8
|
STOPPED = 0
|
9
|
+
# value of @state that indicates the Actor is currently ready to process requests
|
6
10
|
RUNNING = 1
|
11
|
+
# value of @state that indicates the Actor is shutting down
|
7
12
|
CLOSING = 2
|
13
|
+
# value of @state that indicates the Actor is processing one request and
|
14
|
+
# will stop after that processing is complete
|
8
15
|
STEP = 3
|
9
16
|
|
17
|
+
# The current processing state of the Actor
|
18
|
+
attr_reader :state
|
19
|
+
|
20
|
+
# Initializes the Actor with the options specified. A new thread is
|
21
|
+
# created during intialization after calling the set_options() method.
|
22
|
+
# @param [Hash] options options to be used for initialization
|
23
|
+
# @option options [Fixnum] :max_queue_count maximum number of requests
|
24
|
+
# that can be queued before backpressure is applied to the caller.
|
25
|
+
# @option options [Float] :ask_timeout timeout in seconds to wait
|
26
|
+
# before raising a BusyError if the request queue if too long.
|
10
27
|
def initialize(options={})
|
11
28
|
@queue = []
|
12
29
|
@idle = []
|
@@ -55,18 +72,29 @@ module Opee
|
|
55
72
|
sleep(1.0)
|
56
73
|
end
|
57
74
|
rescue Exception => e
|
58
|
-
Env.
|
75
|
+
Env.rescue(e)
|
59
76
|
end
|
60
77
|
end
|
61
78
|
end
|
62
79
|
end
|
63
80
|
|
81
|
+
# Processes the initialize() options. Subclasses should call super.
|
82
|
+
# @param [Hash] options options to be used for initialization
|
83
|
+
# @option options [Fixnum] :max_queue_count maximum number of requests
|
84
|
+
# that can be queued before backpressure is applied to the caller.
|
85
|
+
# @option options [Float] :ask_timeout timeout in seconds to wait
|
86
|
+
# before raising a BusyError if the request queue is too long.
|
64
87
|
def set_options(options)
|
65
88
|
@max_queue_count = options.fetch(:max_queue_count, @max_queue_count)
|
66
|
-
@ask_timeout = options.fetch(:ask_timeout, @ask_timeout)
|
89
|
+
@ask_timeout = options.fetch(:ask_timeout, @ask_timeout).to_f
|
67
90
|
end
|
68
91
|
|
69
|
-
#
|
92
|
+
# Calls {#ask}() but uses the specified timeout instead of the default
|
93
|
+
# {#ask_timeout} to determine how long to wait if the Actor's queue is full.
|
94
|
+
# @param [Fixnum|Float] timeout maximum time to wait trying to add a request to the Actor's queue
|
95
|
+
# @param [Symbol] op method to queue for the Actor
|
96
|
+
# @param [Array] *args arguments to the op method
|
97
|
+
# @raise [BusyError] if the request queue does not become available in the timeout specified
|
70
98
|
def timeout_ask(timeout, op, *args)
|
71
99
|
unless @max_queue_count.nil? || 0 == @max_queue_count || @queue.size() < @max_queue_count
|
72
100
|
@ask_thread = Thread.current
|
@@ -80,10 +108,18 @@ module Opee
|
|
80
108
|
@loop.wakeup() if RUNNING == @state
|
81
109
|
end
|
82
110
|
|
111
|
+
# Queues an operation and arguments to be called when the Actor is ready.
|
112
|
+
# @param [Symbol] op method to queue for the Actor
|
113
|
+
# @param [Array] *args arguments to the op method
|
114
|
+
# @raise [BusyError] if the request queue does not become available in the {#ask_timeout} seconds
|
83
115
|
def ask(op, *args)
|
84
116
|
timeout_ask(@ask_timeout, op, *args)
|
85
117
|
end
|
86
118
|
|
119
|
+
# Queues an operation and arguments to be called when the Actor is has no
|
120
|
+
# other requests to process.
|
121
|
+
# @param [Symbol] op method to queue for the Actor
|
122
|
+
# @param [Array] *args arguments to the op method
|
87
123
|
def on_idle(op, *args)
|
88
124
|
@idle_mutex.synchronize {
|
89
125
|
@idle.insert(0, Act.new(op, args))
|
@@ -91,6 +127,10 @@ module Opee
|
|
91
127
|
@loop.wakeup() if RUNNING == @state
|
92
128
|
end
|
93
129
|
|
130
|
+
# Queues an operation and arguments to be called as soon as possible by
|
131
|
+
# the Actor. These requests take precedence over other ordinary requests.
|
132
|
+
# @param [Symbol] op method to queue for the Actor
|
133
|
+
# @param [Array] *args arguments to the op method
|
94
134
|
def priority_ask(op, *args)
|
95
135
|
@priority_mutex.synchronize {
|
96
136
|
@priority.insert(0, Act.new(op, args))
|
@@ -98,47 +138,74 @@ module Opee
|
|
98
138
|
@loop.wakeup() if RUNNING == @state
|
99
139
|
end
|
100
140
|
|
141
|
+
# When an attempt is made to call a private method of the Actor it is
|
142
|
+
# places on the processing queue. Other methods cause a NoMethodError to
|
143
|
+
# be raised as it normally would.
|
144
|
+
# @param [Symbol] m method to queue for the Actor
|
145
|
+
# @param [Array] *args arguments to the op method
|
146
|
+
# @param [Proc] blk ignored
|
101
147
|
def method_missing(m, *args, &blk)
|
102
148
|
raise NoMethodError.new("undefine method '#{m}' for #{self.class}", m, args) unless respond_to?(m, true)
|
103
149
|
ask(m, *args)
|
104
150
|
end
|
105
151
|
|
152
|
+
# Returns the number of requests on all three request queues, the normal,
|
153
|
+
# priority, and idle queues.
|
154
|
+
# @return [Fixnum] number of queued requests
|
106
155
|
def queue_count()
|
107
156
|
@queue.length + @priority.length + @idle.length
|
108
157
|
end
|
109
158
|
|
110
|
-
|
159
|
+
# Returns the true if any requests are queued or a request is being processed.
|
160
|
+
# @return [true|false] true if busy, false otherwise
|
161
|
+
def busy?()
|
111
162
|
@busy || !@queue.empty? || !@priority.empty? || !@idle.empty?
|
112
163
|
end
|
113
164
|
|
165
|
+
# Returns the default timeout for the time to wait for the Actor to be
|
166
|
+
# ready to accept a request using the {#ask}() method.
|
167
|
+
# @return [Float] current timeout for the {#ask}() method
|
114
168
|
def ask_timeout()
|
115
169
|
@ask_timeout
|
116
170
|
end
|
117
171
|
|
118
|
-
|
172
|
+
# Returns the maximum number of requests allowed on the normal processing
|
173
|
+
# queue. A value of nil indicates there is no limit.
|
174
|
+
# @return [NilClass|Fixnum] maximum number of request that can be queued
|
175
|
+
def max_queue_count()
|
119
176
|
@max_queue_count
|
120
177
|
end
|
121
178
|
|
179
|
+
# Causes the Actor to stop processing any more requests after the current
|
180
|
+
# request has finished.
|
122
181
|
def stop()
|
123
182
|
@state = STOPPED
|
124
183
|
end
|
125
184
|
|
185
|
+
# Causes the Actor to process one request and then stop. The max_wait is
|
186
|
+
# used to avoid getting stuck if the processing takes too long.
|
187
|
+
# @param [Float|Fixnum] max_wait maximum time to wait for the step to complete
|
126
188
|
def step(max_wait=5)
|
127
189
|
@state = STEP
|
128
190
|
@step_thread = Thread.current
|
129
191
|
@loop.wakeup()
|
130
192
|
sleep(max_wait)
|
193
|
+
@step_thread = nil
|
131
194
|
end
|
132
195
|
|
196
|
+
# Wakes up the Actor if it has been stopped or if Env.shutdown() has been called.
|
133
197
|
def wakeup()
|
134
198
|
@loop.wakeup()
|
135
199
|
end
|
136
200
|
|
201
|
+
# Restarts the Actor's processing thread.
|
137
202
|
def start()
|
138
203
|
@state = RUNNING
|
139
204
|
@loop.wakeup()
|
140
205
|
end
|
141
206
|
|
207
|
+
# Closes the Actor by exiting the processing thread and removing the Actor
|
208
|
+
# from the Env.
|
142
209
|
def close()
|
143
210
|
@state = CLOSING
|
144
211
|
begin
|
@@ -153,14 +220,24 @@ module Opee
|
|
153
220
|
|
154
221
|
private
|
155
222
|
|
223
|
+
# Sets the {#ask_timeout} value which limits how long a caller will wait
|
224
|
+
# before getting a BusyError when calling {#ask}(). This method is
|
225
|
+
# executed asynchronously.
|
226
|
+
# @param [Float] timeout seconds to set the ask_timeout to
|
156
227
|
def ask_timeout=(timeout)
|
157
228
|
@ask_timeout = timeout
|
158
229
|
end
|
159
230
|
|
231
|
+
# Sets the {#max_queue_count} value which limits the number of requests
|
232
|
+
# that can be on the Actor's queue. This method is executed
|
233
|
+
# asynchronously.
|
234
|
+
# @param [Fixnum] max maximum number of requests or nil for no limit
|
160
235
|
def max_queue_count=(max)
|
161
236
|
@max_queue_count = max
|
162
237
|
end
|
163
238
|
|
239
|
+
# Internal class used to store information about asynchronous method
|
240
|
+
# invocations.
|
164
241
|
class Act
|
165
242
|
attr_accessor :op
|
166
243
|
attr_accessor :args
|
@@ -0,0 +1,103 @@
|
|
1
|
+
|
2
|
+
module Opee
|
3
|
+
# This class is used to collect multiple paths together before proceeding in
|
4
|
+
# a system. Two approaches are supported, either let the Collector subclass
|
5
|
+
# identify when it is time to move on or put the logic in the Job that is
|
6
|
+
# passed between the various Actors.
|
7
|
+
#
|
8
|
+
# To use a Collector, pass the same data along each path of Actors. All
|
9
|
+
# paths should terminate at the Collector. From there the Collector keeps a
|
10
|
+
# cache of the data's key and a token that is used to track arrivals. When
|
11
|
+
# all paths have converged the next Actor in the process is called.
|
12
|
+
class Collector < Actor
|
13
|
+
|
14
|
+
def initialize(options={})
|
15
|
+
@cache = {}
|
16
|
+
@next_actor = nil
|
17
|
+
@next_method = nil
|
18
|
+
super(options)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Returns the number of Jobs currently waiting to be matched.
|
22
|
+
# @return [Fixnum] current number of Jobs waiting to finish.
|
23
|
+
def cache_size()
|
24
|
+
@cache.size()
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
# Processes the initialize() options. Subclasses should call super.
|
30
|
+
# @param [Hash] options options to be used for initialization
|
31
|
+
# @option options [Actor] :next_actor Actor to ask to continue when ready
|
32
|
+
# @option options [Symbol] :next_method method to ask of the next_actor to continue when ready
|
33
|
+
def set_options(options)
|
34
|
+
super(options)
|
35
|
+
@next_actor = options[:next_actor]
|
36
|
+
@next_method = options[:next_method]
|
37
|
+
@next_method = @next_method.to_sym if @next_method.is_a?(String)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Collects a job and deternines if the job should be moved on to the next
|
41
|
+
# Actor or if it should wait until more processing paths have
|
42
|
+
# finished. This method is executed asynchronously.
|
43
|
+
# @param [Job|Object] job data to process or pass on
|
44
|
+
def collect(job, path_id=nil)
|
45
|
+
key = job_key(job)
|
46
|
+
token = @cache[key]
|
47
|
+
token = update_token(job, token, path_id)
|
48
|
+
if complete?(job, token)
|
49
|
+
@cache.delete(key)
|
50
|
+
keep_going(job)
|
51
|
+
else
|
52
|
+
@cache[key] = token
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns the key associated with the job. If the job responds to :key
|
57
|
+
# then that method is called, otherwise the subclass should implement this
|
58
|
+
# method. This method is executed asynchronously.
|
59
|
+
# @param [Object] job data to get the key for
|
60
|
+
# @return [Object] a key for looking up the token in the cache
|
61
|
+
def job_key(job)
|
62
|
+
raise NotImplementedError.new("neither Collector.job_key() nor Job.key() are implemented") unless job.respond_to?(:key)
|
63
|
+
job.key()
|
64
|
+
end
|
65
|
+
|
66
|
+
# Updates the token associated with the job. The job or the Collector
|
67
|
+
# subclass can use any data desired to keep track of the job's paths that
|
68
|
+
# have been completed. This method is executed asynchronously.
|
69
|
+
# @param [Object] job data to get the key for
|
70
|
+
# @param [Object] token current token value or nil for the first token value
|
71
|
+
# @param [Object] path_id an indicator of the path if used
|
72
|
+
# @return [Object] a token to keep track of the progress of the job
|
73
|
+
def update_token(job, token, path_id)
|
74
|
+
raise NotImplementedError.new("neither Collector.update_token() nor Job.update_token() are implemented") unless job.respond_to?(:update_token)
|
75
|
+
job.update_token(token, path_id)
|
76
|
+
end
|
77
|
+
|
78
|
+
# Returns true if the job has been processed by all paths converging on
|
79
|
+
# the collector. This can be implemented in the Collector subclass or in
|
80
|
+
# the Job. This method is executed asynchronously.
|
81
|
+
# @param [Object] job data to get the key for
|
82
|
+
# @param [Object] token current token value or nil for the first token value
|
83
|
+
# @return [true|false] an indication of wether the job has completed all paths
|
84
|
+
def complete?(job, token)
|
85
|
+
raise NotImplementedError.new("neither Collector.complete?() nor Job.complete?() are implemented") unless job.respond_to?(:complete?)
|
86
|
+
job.complete?(token)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Moves the job onto the next Actor. If the job responds to :keep_going
|
90
|
+
# that is used, otherwise the @next_actor and @next_method care used to
|
91
|
+
# continue. This method is executed asynchronously.
|
92
|
+
# @param [Object] job data to get the key for
|
93
|
+
def keep_going(job)
|
94
|
+
if job.respond_to?(:keep_going)
|
95
|
+
job.keep_going()
|
96
|
+
else
|
97
|
+
# TBD @next_actor = Env.find_actor(@next_actor) if @next_actor.is_a?(Symbol)
|
98
|
+
@next_actor.send(@next_method, job)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
end # Collector
|
103
|
+
end # Opee
|
data/lib/opee/env.rb
CHANGED
@@ -1,27 +1,54 @@
|
|
1
1
|
|
2
|
+
# Opee is an experimental Object-based Parallel Evaluation Environment
|
3
|
+
#
|
4
|
+
# The purpose of OPEE is to explore setting up an environment that will run
|
5
|
+
# completely in parallel with minimum use of mutex synchronization. Actors
|
6
|
+
# only push the flow forward, never returning values when using the ask()
|
7
|
+
# method. Other methods return immediately but they should never modify the
|
8
|
+
# data portions of the Actors. They can be used to modify the control
|
9
|
+
# parameters of the Actor.
|
2
10
|
module Opee
|
11
|
+
# The Env class hold all the global data for Opee. The global data could
|
12
|
+
# have been put directly under the Opee module but it seemed cleaner to keep
|
13
|
+
# it all together in an separate class.
|
3
14
|
class Env
|
4
|
-
|
15
|
+
# Array of active Actors. This is private and should not be modified
|
16
|
+
# directly.
|
5
17
|
@@actors = []
|
18
|
+
# global logger. This is private and should not be modified directly.
|
6
19
|
@@log = nil
|
20
|
+
# used to wakup calling thread when ready. This is private and should not
|
21
|
+
# be modified directly.
|
7
22
|
@@finish_thread = nil
|
23
|
+
# Actor that responds to rescue if set. This is private and should not be
|
24
|
+
# modified directly. Use the rescuer() or rescuer=() methods instead.
|
25
|
+
@@rescuer = nil
|
8
26
|
|
27
|
+
# Adds an Actor to the Env. This is called by the {Actor#initialize}() method.
|
28
|
+
# @param [Actor] actor actor to add
|
9
29
|
def self.add_actor(actor)
|
10
30
|
@@actors << actor
|
11
31
|
end
|
12
32
|
|
33
|
+
# Removes an Actor from the Env. This is called by the {Actor#close}() method.
|
34
|
+
# @param [Actor] actor actor to remove
|
13
35
|
def self.remove_actor(actor)
|
14
36
|
@@actors.delete(actor)
|
15
37
|
end
|
16
38
|
|
39
|
+
# Iterates over each active Actor and yields to the provided block with
|
40
|
+
# each Actor.
|
41
|
+
# @param [Proc] blk Proc to call on each iteration
|
17
42
|
def self.each_actor(&blk)
|
18
43
|
@@actors.each { |a| blk.yield(a) }
|
19
44
|
end
|
20
45
|
|
46
|
+
# Returns the number of active Actors.
|
21
47
|
def self.actor_count()
|
22
48
|
@@actors.size
|
23
49
|
end
|
24
50
|
|
51
|
+
# Closes all Actors and resets the logger to nil.
|
25
52
|
def self.shutdown()
|
26
53
|
until @@actors.empty?
|
27
54
|
a = @@actors.pop()
|
@@ -30,31 +57,67 @@ module Opee
|
|
30
57
|
@@log = nil
|
31
58
|
end
|
32
59
|
|
60
|
+
# Asks the logger to log the message if the severity is high enough.
|
61
|
+
# @param [Fixnum] severity one of the Logger levels
|
62
|
+
# @param [String] message string to log
|
33
63
|
def self.log(severity, message)
|
34
64
|
@@log = Log.new() if @@log.nil?
|
35
65
|
@@log.ask(:log, severity, message)
|
36
66
|
end
|
37
67
|
|
68
|
+
# Asks the logger to log a message if the current severity level is less
|
69
|
+
# than or equal to Logger::DEBUG.
|
70
|
+
# @param [String] message string to log
|
38
71
|
def self.debug(message)
|
39
72
|
log(Logger::Severity::DEBUG, message)
|
40
73
|
end
|
41
74
|
|
75
|
+
# Asks the logger to log a message if the current severity level is less
|
76
|
+
# than or equal to Logger::INFO.
|
77
|
+
# @param [String] message string to log
|
42
78
|
def self.info(message)
|
43
79
|
log(Logger::Severity::INFO, message)
|
44
80
|
end
|
45
81
|
|
82
|
+
# Asks the logger to log a message if the current severity level is less
|
83
|
+
# than or equal to Logger::WARN.
|
84
|
+
# @param [String] message string to log
|
46
85
|
def self.warn(message)
|
47
86
|
log(Logger::Severity::WARN, message)
|
48
87
|
end
|
49
88
|
|
89
|
+
# Asks the logger to log a message if the current severity level is less
|
90
|
+
# than or equal to Logger::ERROR.
|
91
|
+
# @param [String] message string to log
|
50
92
|
def self.error(message)
|
51
93
|
log(Logger::Severity::ERROR, message)
|
52
94
|
end
|
53
95
|
|
96
|
+
# Asks the logger to log a message if the current severity level is less
|
97
|
+
# than or equal to Logger::FATAL.
|
98
|
+
# @param [String] message string to log
|
54
99
|
def self.fatal(message)
|
55
100
|
log(Logger::Severity::FATAL, message)
|
56
101
|
end
|
57
102
|
|
103
|
+
# The log_rescue() method is called and then If a rescuer is set then it's
|
104
|
+
# rescue() method is called.
|
105
|
+
# @param [Exception] ex Exception to handle
|
106
|
+
def self.rescue(ex)
|
107
|
+
begin
|
108
|
+
log_rescue(ex)
|
109
|
+
@@rescuer.rescue(ex) unless @@rescuer.nil?
|
110
|
+
rescue Exception => e
|
111
|
+
puts "*** #{e.class}: #{e.message}"
|
112
|
+
e.backtrace.each { |line| puts " " + line }
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Asks the logger to log a Exception class and message if the current
|
117
|
+
# severity level is less than or equal to Logger::ERROR. If the current
|
118
|
+
# severity is less than or equal to Logger::WARN then the Exception
|
119
|
+
# bactrace is also logged.
|
120
|
+
# @param [Exception] ex Exception to log
|
58
121
|
def self.log_rescue(ex)
|
59
122
|
@@log = Log.new() if @@log.nil?
|
60
123
|
return unless Logger::Severity::ERROR >= @@log.level
|
@@ -65,37 +128,67 @@ module Opee
|
|
65
128
|
@@log.ask(:log, Logger::Severity::ERROR, msg)
|
66
129
|
end
|
67
130
|
|
131
|
+
# Returns the current Log Actor. If the current logger is nil then a new
|
132
|
+
# Log Actor is created and returned.
|
133
|
+
# @return [Log] the current logger
|
68
134
|
def self.logger()
|
69
135
|
@@log = Log.new() if @@log.nil?
|
70
136
|
@@log
|
71
137
|
end
|
72
138
|
|
139
|
+
# Sets the logger to the provided Log Actor. If the current logger is not
|
140
|
+
# nil then the current logger is closed first.
|
141
|
+
# @param [Log] log_actor Log Actor to use as the logger
|
142
|
+
# @raise [TypeError] raised if the log_actor is not a Log Actor
|
73
143
|
def self.logger=(log_actor)
|
144
|
+
raise TypeError.new("can't convert #{log_actor.class} into a Opee::Log") unless log_actor.is_a?(Log)
|
74
145
|
@@log.close() unless @@log.nil?
|
75
146
|
@@log = log_actor
|
76
147
|
@@log
|
77
148
|
end
|
78
149
|
|
150
|
+
# Returns the current rescuer Actor. The rescuer us the Actor that is
|
151
|
+
# sent an Exception if an Exception is raised by an Actor.
|
152
|
+
# @return [Actor] the current rescuer
|
153
|
+
def self.rescuer()
|
154
|
+
@@rescuer
|
155
|
+
end
|
156
|
+
|
157
|
+
# Sets the rescuer to the provided Actor.
|
158
|
+
# @param [Actor] actor Actor to use as the rescuer and responds to :rescue
|
159
|
+
def self.rescuer=(actor)
|
160
|
+
@@rescuer = actor
|
161
|
+
end
|
162
|
+
|
163
|
+
# Returns the sum of all the requests in all the Actor's queues.
|
164
|
+
# @return [Fixnum] total number of items waiting to be processed
|
79
165
|
def self.queue_count()
|
80
166
|
cnt = 0
|
81
167
|
@@actors.each { |a| cnt += a.queue_count() }
|
82
168
|
cnt
|
83
169
|
end
|
84
170
|
|
171
|
+
# Returns true of one or more Actors is either processing a request or has
|
172
|
+
# a request waiting to be processed on it's input queue.
|
173
|
+
# @return [true|false] the busy state across all Actors
|
85
174
|
def self.busy?
|
86
175
|
@@actors.each { |a| return true if a.busy? }
|
87
176
|
false
|
88
177
|
end
|
89
178
|
|
179
|
+
# Calls the stop() method on all Actors.
|
90
180
|
def self.stop()
|
91
181
|
@@actors.each { |a| a.stop() }
|
92
182
|
end
|
93
183
|
|
184
|
+
# Calls the start() method on all Actors.
|
94
185
|
def self.start()
|
95
186
|
@@finish_thread = nil
|
96
187
|
@@actors.each { |a| a.start() }
|
97
188
|
end
|
98
189
|
|
190
|
+
# Waits for all Actors to complete processing. The method only returns
|
191
|
+
# when all Actors indicate they are no longer busy.
|
99
192
|
def self.wait_finish()
|
100
193
|
@@finish_thread = Thread.current
|
101
194
|
@@actors.each { |a| a.wakeup() }
|
@@ -104,10 +197,13 @@ module Opee
|
|
104
197
|
end
|
105
198
|
end
|
106
199
|
|
200
|
+
# Wakes up the calling thread when an Actor is finished. It is called by
|
201
|
+
# the Actor and should not be called by any other code.
|
107
202
|
def self.wake_finish()
|
108
203
|
@@finish_thread.wakeup() unless @@finish_thread.nil?
|
109
204
|
end
|
110
205
|
|
206
|
+
# Waits until all Actors are not longer busy and then closes all Actors.
|
111
207
|
def self.wait_close()
|
112
208
|
while 0 < queue_count()
|
113
209
|
wait_finish()
|
data/lib/opee/errors.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
|
2
2
|
module Opee
|
3
|
+
# An Exception indicating an required option was missing.
|
3
4
|
class MissingOptionError < Exception
|
4
5
|
def initialize(option, msg)
|
5
6
|
super("option #{option} for #{msg} missing")
|
6
7
|
end
|
7
8
|
end # MissingOptionError
|
8
9
|
|
10
|
+
# An Exception indicating an Actor was too busy to complete the requested operation.
|
9
11
|
class BusyError < Exception
|
10
12
|
def initialize()
|
11
13
|
super("Busy, try again later")
|
data/lib/opee/job.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
module Opee
|
3
|
+
# A holder for data processed by Actors. It lends support to the use of
|
4
|
+
# Collectors if the support behavior is desired in the Job instead of the
|
5
|
+
# Collector itself.
|
6
|
+
class Job
|
7
|
+
|
8
|
+
def initialize()
|
9
|
+
end
|
10
|
+
|
11
|
+
# Returns the key associated with the job. The default behavior is to
|
12
|
+
# raise a NotImplementedError.
|
13
|
+
# @return [Object] a key for looking up the token in the cache
|
14
|
+
def key()
|
15
|
+
object_id()
|
16
|
+
end
|
17
|
+
|
18
|
+
# Updates the token associated with the job. The default behavior is to
|
19
|
+
# raise a NotImplementedError.
|
20
|
+
# @param [Object] token current token value or nil for the first token value
|
21
|
+
# @param [Object] path_id an indicator of the path if used
|
22
|
+
# @return [Object] a token to keep track of the progress of the job
|
23
|
+
def update_token(token, path_id)
|
24
|
+
raise NotImplementedError.new("update_token() not implemented")
|
25
|
+
end
|
26
|
+
|
27
|
+
# Returns true if the job has been processed by all paths converging on
|
28
|
+
# the collector. The default behavior is to raise a NotImplementedError.
|
29
|
+
# @param [Object] token current token value or nil for the first token value
|
30
|
+
# @return [true|false] an indication of wether the job has completed all paths
|
31
|
+
def complete?(token)
|
32
|
+
raise NotImplementedError.new("complete?() not implemented")
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
# Moves the job onto the next Actor. Make public if implemented otherwise
|
38
|
+
# leave private so it is not called.
|
39
|
+
def keep_going()
|
40
|
+
end
|
41
|
+
|
42
|
+
end # Job
|
43
|
+
end # Opee
|
data/lib/opee/log.rb
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
require 'logger'
|
3
3
|
|
4
4
|
module Opee
|
5
|
+
# An asynchronous logger build on top of the Actor class. It is able to log
|
6
|
+
# messages as well as forward calls to a {#forward} Actor.
|
5
7
|
class Log < Actor
|
6
8
|
|
7
9
|
def initialize(options={})
|
@@ -10,14 +12,20 @@ module Opee
|
|
10
12
|
super(options)
|
11
13
|
end
|
12
14
|
|
15
|
+
# Returns the current severity level.
|
16
|
+
# @return [Fixnum] Logger severity level
|
13
17
|
def level()
|
14
18
|
@logger.level
|
15
19
|
end
|
16
20
|
|
21
|
+
# Returns the current formatter.
|
22
|
+
# @return [Logger::Formatter] current formatter
|
17
23
|
def formatter()
|
18
24
|
@logger.formatter
|
19
25
|
end
|
20
26
|
|
27
|
+
# Returns the forward Log Actor if one has been set.
|
28
|
+
# @return [Log] the forward Log Actor if set
|
21
29
|
def forward()
|
22
30
|
@forward
|
23
31
|
end
|
@@ -26,6 +34,14 @@ module Opee
|
|
26
34
|
# Use ask() to invoke private methods. If called directly they will be
|
27
35
|
# picked up by the Actor method_missing() method.
|
28
36
|
|
37
|
+
# Sets the logger, severity, and formatter if provided.
|
38
|
+
# @param [Hash] options options to be used for initialization
|
39
|
+
# @option options [String] :filename filename to write to
|
40
|
+
# @option options [Fixnum] :max_file_size maximum file size
|
41
|
+
# @option options [Fixnum] :max_file_count maximum number of log file
|
42
|
+
# @option options [IO] :stream IO stream
|
43
|
+
# @option options [String|Fixnum] :severity initial setting for severity
|
44
|
+
# @option options [Proc] :formatter initial setting for the formatter procedure
|
29
45
|
def set_options(options)
|
30
46
|
super(options)
|
31
47
|
if !(filename = options[:filename]).nil?
|
@@ -41,11 +57,18 @@ module Opee
|
|
41
57
|
formatter = options[:formatter] if options.has_key?(:formatter)
|
42
58
|
end
|
43
59
|
|
60
|
+
# Writes a message if the severity is high enough. This method is
|
61
|
+
# executed asynchronously.
|
62
|
+
# @param [Fixnum] severity one of the Logger levels
|
63
|
+
# @param [String] message string to log
|
44
64
|
def log(severity, message)
|
45
65
|
@logger.add(severity, message)
|
46
66
|
@forward.log(severity, message) unless @forward.nil?
|
47
67
|
end
|
48
68
|
|
69
|
+
# Sets the logger to use the stream specified. This method is executed
|
70
|
+
# asynchronously.
|
71
|
+
# @param [IO] stream stream to write log messages to
|
49
72
|
def stream=(stream)
|
50
73
|
logger = Logger.new(stream)
|
51
74
|
logger.level = @logger.level
|
@@ -53,6 +76,11 @@ module Opee
|
|
53
76
|
@logger = logger
|
54
77
|
end
|
55
78
|
|
79
|
+
# Creates a new Logger to write log messages to using the parameters
|
80
|
+
# specified. This method is executed asynchronously.
|
81
|
+
# @param [String] filename filename of active log file
|
82
|
+
# @param [Fixmun] shift_age maximum number of archive files to save
|
83
|
+
# @param [Fixmun] shift_size maximum file size
|
56
84
|
def set_filename(filename, shift_age=7, shift_size=1048576)
|
57
85
|
logger = Logger.new(filename, shift_age, shift_size)
|
58
86
|
logger.level = @logger.level
|
@@ -60,14 +88,23 @@ module Opee
|
|
60
88
|
@logger = logger
|
61
89
|
end
|
62
90
|
|
91
|
+
# Replace the logger with a new Logger Object. This method is executed
|
92
|
+
# asynchronously.
|
93
|
+
# @param [Logger] logger replacement logger
|
63
94
|
def logger=(logger)
|
64
95
|
@logger = logger
|
65
96
|
end
|
66
97
|
|
98
|
+
# Sets the {#forward} to the actor specified. This method is executed
|
99
|
+
# asynchronously.
|
100
|
+
# @param [Log] forward_actor Log Actor to forward log calls to
|
67
101
|
def forward=(forward_actor)
|
68
102
|
@forward = forward_actor
|
69
103
|
end
|
70
104
|
|
105
|
+
# Sets the severity level of the logger. This method is executed
|
106
|
+
# asynchronously.
|
107
|
+
# @param [String|Fixnum] level value to set the severity to
|
71
108
|
def severity=(level)
|
72
109
|
if level.is_a?(String)
|
73
110
|
severity = {
|
@@ -85,6 +122,9 @@ module Opee
|
|
85
122
|
@logger.level = level
|
86
123
|
end
|
87
124
|
|
125
|
+
# Sets the formatter procedure of the logger. This method is executed
|
126
|
+
# asynchronously.
|
127
|
+
# @param [Proc] proc value to set the formatter to
|
88
128
|
def formatter=(proc)
|
89
129
|
@logger.formatter = proc
|
90
130
|
end
|
data/lib/opee/version.rb
CHANGED
data/lib/opee/workqueue.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
|
2
2
|
module Opee
|
3
|
+
# Implements a work queue Actor that ill distribute jobs to Actors that
|
4
|
+
# volunteer to complete those jobs. The primary use is to distribute work or
|
5
|
+
# jobs across multiple workers.
|
3
6
|
class WorkQueue < Actor
|
4
7
|
|
5
8
|
def initialize(options={})
|
@@ -12,18 +15,34 @@ module Opee
|
|
12
15
|
super(options)
|
13
16
|
end
|
14
17
|
|
18
|
+
# Returns the number of jobs currently on the work queue.
|
19
|
+
# @return [Fixnum] number of waiting jobs
|
15
20
|
def work_queue_size()
|
16
21
|
@work_queue.size
|
17
22
|
end
|
18
23
|
|
24
|
+
# Returns the number of worker Actors waiting to process jobs.
|
25
|
+
# @return [Fixnum] number of waiting workers
|
19
26
|
def worker_count()
|
20
27
|
@workers.size
|
21
28
|
end
|
22
29
|
|
30
|
+
# Returns the method invoked on the workers to process a job.
|
31
|
+
# @return [Symbol] method workers are asked to complete
|
32
|
+
def method()
|
33
|
+
@workers.size
|
34
|
+
end
|
35
|
+
|
36
|
+
# Returns the true if any requests are queued, a request is being
|
37
|
+
# processed, or if there are jobs waiting on the work request queue.
|
38
|
+
# @return [true|false] true if busy, false otherwise
|
23
39
|
def busy?
|
24
40
|
!@work_queue.empty? || super
|
25
41
|
end
|
26
42
|
|
43
|
+
# Verifies that additional jobs can be added to the work queue before
|
44
|
+
# allowing an {#add}() to be called.
|
45
|
+
# @see Actor#ask
|
27
46
|
def ask(op, *args)
|
28
47
|
if :add == op && 0 < @max_job_count && (@work_queue.size() + @queue.size()) >= @max_job_count
|
29
48
|
unless 0.0 >= @job_timeout
|
@@ -41,6 +60,13 @@ module Opee
|
|
41
60
|
|
42
61
|
private
|
43
62
|
|
63
|
+
# Processes the initialize() options. Subclasses should call super.
|
64
|
+
# @param [Hash] options options to be used for initialization
|
65
|
+
# @option options [Symbol] :method method to call on workers
|
66
|
+
# @option options [Fixnum] :max_job_count maximum number of jobs
|
67
|
+
# that can be queued before backpressure is applied to the caller.
|
68
|
+
# @option options [Float] :job_timeout timeout in seconds to wait
|
69
|
+
# before raising a BusyError if the work queue is too long.
|
44
70
|
def set_options(options)
|
45
71
|
super(options)
|
46
72
|
raise MissingOptionError.new(:method, "for processing jobs") if (@method = options[:method]).nil?
|
@@ -48,6 +74,8 @@ module Opee
|
|
48
74
|
@job_timeout = options.fetch(:job_timeout, @job_timeout)
|
49
75
|
end
|
50
76
|
|
77
|
+
# Places a job on the work queue. This method is executed asynchronously.
|
78
|
+
# @param [Object] job work to be processed
|
51
79
|
def add(job)
|
52
80
|
if @workers.empty?
|
53
81
|
@work_queue.insert(0, job)
|
@@ -57,6 +85,9 @@ module Opee
|
|
57
85
|
end
|
58
86
|
end
|
59
87
|
|
88
|
+
# Identifies a worker as available to process jobs when they become
|
89
|
+
# available. This method is executed asynchronously.
|
90
|
+
# @param [Actor] worker Actor that responds to the {#method}
|
60
91
|
def ready(worker)
|
61
92
|
if @work_queue.empty?
|
62
93
|
@workers.insert(0, worker) unless @workers.include?(worker)
|
@@ -0,0 +1,152 @@
|
|
1
|
+
#!/usr/bin/env ruby -wW2
|
2
|
+
# encoding: UTF-8
|
3
|
+
|
4
|
+
[ File.dirname(__FILE__),
|
5
|
+
File.join(File.dirname(__FILE__), "../lib")
|
6
|
+
].each { |path| $: << path unless $:.include?(path) }
|
7
|
+
|
8
|
+
require 'test/unit'
|
9
|
+
require 'opee'
|
10
|
+
|
11
|
+
class NotJob
|
12
|
+
attr_reader :a, :b, :c
|
13
|
+
|
14
|
+
def initialize()
|
15
|
+
@a = []
|
16
|
+
@b = []
|
17
|
+
@c = []
|
18
|
+
end
|
19
|
+
|
20
|
+
end # NotJob
|
21
|
+
|
22
|
+
class NiceJob < ::Opee::Job
|
23
|
+
attr_reader :a, :b, :c
|
24
|
+
|
25
|
+
def initialize(next_actor)
|
26
|
+
@next_actor = next_actor
|
27
|
+
@a = []
|
28
|
+
@b = []
|
29
|
+
@c = []
|
30
|
+
end
|
31
|
+
|
32
|
+
def update_token(token, path_id)
|
33
|
+
token = [] if token.nil?
|
34
|
+
token << path_id
|
35
|
+
token
|
36
|
+
end
|
37
|
+
|
38
|
+
def complete?(token)
|
39
|
+
3 <= token.size && token.include?(:a) && token.include?(:b) && token.include?(:c)
|
40
|
+
end
|
41
|
+
|
42
|
+
def keep_going()
|
43
|
+
@next_actor.ask(:done, self)
|
44
|
+
end
|
45
|
+
|
46
|
+
end # NiceJob
|
47
|
+
|
48
|
+
class Fan < ::Opee::Actor
|
49
|
+
private
|
50
|
+
|
51
|
+
def set_options(options)
|
52
|
+
super(options)
|
53
|
+
@fans = options[:fans]
|
54
|
+
end
|
55
|
+
|
56
|
+
def go(job)
|
57
|
+
@fans.each { |f| f.ask(:set, job) }
|
58
|
+
end
|
59
|
+
|
60
|
+
end # Fan
|
61
|
+
|
62
|
+
class Setter < ::Opee::Actor
|
63
|
+
private
|
64
|
+
|
65
|
+
def set_options(options)
|
66
|
+
super(options)
|
67
|
+
@attr = options[:attr]
|
68
|
+
@forward = options[:forward]
|
69
|
+
end
|
70
|
+
|
71
|
+
def set(job)
|
72
|
+
job.send(@attr) << true
|
73
|
+
@forward.ask(:collect, job, @attr)
|
74
|
+
end
|
75
|
+
|
76
|
+
end # Setter
|
77
|
+
|
78
|
+
class Col < ::Opee::Collector
|
79
|
+
private
|
80
|
+
|
81
|
+
def set_options(options)
|
82
|
+
super(options)
|
83
|
+
@done = options[:done]
|
84
|
+
end
|
85
|
+
|
86
|
+
def job_key(job)
|
87
|
+
job.object_id()
|
88
|
+
end
|
89
|
+
|
90
|
+
def update_token(job, token, path_id)
|
91
|
+
token.to_i + 1
|
92
|
+
end
|
93
|
+
|
94
|
+
def complete?(job, token)
|
95
|
+
3 <= token
|
96
|
+
end
|
97
|
+
|
98
|
+
end # Col
|
99
|
+
|
100
|
+
class Done < ::Opee::Actor
|
101
|
+
attr_reader :finished
|
102
|
+
|
103
|
+
def initialize(options={})
|
104
|
+
@finished = false
|
105
|
+
super
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
def done(job)
|
111
|
+
@finished = true
|
112
|
+
end
|
113
|
+
|
114
|
+
end # Done
|
115
|
+
|
116
|
+
class OpeeTest < ::Test::Unit::TestCase
|
117
|
+
|
118
|
+
def test_opee_collector
|
119
|
+
done = Done.new()
|
120
|
+
col = Col.new(:next_actor => done, :next_method => :done)
|
121
|
+
a = Setter.new(:attr => :a, :forward => col)
|
122
|
+
b = Setter.new(:attr => :b, :forward => col)
|
123
|
+
c = Setter.new(:attr => :c, :forward => col)
|
124
|
+
fan = Fan.new(:fans => [a, b, c])
|
125
|
+
job = NotJob.new()
|
126
|
+
fan.ask(:go, job)
|
127
|
+
::Opee::Env.wait_close()
|
128
|
+
assert_equal(true, done.finished)
|
129
|
+
assert_equal(0, col.cache_size)
|
130
|
+
assert_equal([true], job.a)
|
131
|
+
assert_equal([true], job.b)
|
132
|
+
assert_equal([true], job.c)
|
133
|
+
end
|
134
|
+
|
135
|
+
def test_opee_job
|
136
|
+
done = Done.new()
|
137
|
+
col = ::Opee::Collector.new()
|
138
|
+
a = Setter.new(:attr => :a, :forward => col)
|
139
|
+
b = Setter.new(:attr => :b, :forward => col)
|
140
|
+
c = Setter.new(:attr => :c, :forward => col)
|
141
|
+
fan = Fan.new(:fans => [a, b, c])
|
142
|
+
job = NiceJob.new(done)
|
143
|
+
fan.ask(:go, job)
|
144
|
+
::Opee::Env.wait_close()
|
145
|
+
assert_equal(true, done.finished)
|
146
|
+
assert_equal(0, col.cache_size)
|
147
|
+
assert_equal([true], job.a)
|
148
|
+
assert_equal([true], job.b)
|
149
|
+
assert_equal([true], job.c)
|
150
|
+
end
|
151
|
+
|
152
|
+
end # OpeeTest
|
data/test/ts_opee.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: opee
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-05-
|
12
|
+
date: 2012-05-14 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: ! 'An experimental Object-base Parallel Evaluation Environment. '
|
15
15
|
email: peter@ohler.com
|
@@ -19,14 +19,17 @@ extra_rdoc_files:
|
|
19
19
|
- README.md
|
20
20
|
files:
|
21
21
|
- lib/opee/actor.rb
|
22
|
+
- lib/opee/collector.rb
|
22
23
|
- lib/opee/env.rb
|
23
24
|
- lib/opee/errors.rb
|
25
|
+
- lib/opee/job.rb
|
24
26
|
- lib/opee/log.rb
|
25
27
|
- lib/opee/version.rb
|
26
28
|
- lib/opee/workqueue.rb
|
27
29
|
- lib/opee.rb
|
28
30
|
- test/relay.rb
|
29
31
|
- test/tc_opee_actor.rb
|
32
|
+
- test/tc_opee_collector.rb
|
30
33
|
- test/tc_opee_env.rb
|
31
34
|
- test/tc_opee_log.rb
|
32
35
|
- test/tc_opee_workqueue.rb
|
@@ -56,8 +59,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
56
59
|
version: '0'
|
57
60
|
requirements: []
|
58
61
|
rubyforge_project: opee
|
59
|
-
rubygems_version: 1.8.
|
62
|
+
rubygems_version: 1.8.21
|
60
63
|
signing_key:
|
61
64
|
specification_version: 3
|
62
65
|
summary: An experimental Object-base Parallel Evaluation Environment.
|
63
66
|
test_files: []
|
67
|
+
has_rdoc: true
|