opee 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +3 -15
- data/lib/opee.rb +1 -0
- data/lib/opee/actor.rb +38 -2
- data/lib/opee/env.rb +6 -1
- data/lib/opee/errors.rb +15 -0
- data/lib/opee/log.rb +1 -0
- data/lib/opee/version.rb +1 -1
- data/lib/opee/workqueue.rb +31 -1
- data/test/relay.rb +7 -0
- data/test/tc_opee_actor.rb +7 -0
- data/test/tc_opee_workqueue.rb +82 -0
- data/test/ts_opee.rb +1 -0
- metadata +5 -4
data/README.md
CHANGED
@@ -16,16 +16,14 @@ Any comments, thoughts, or suggestions are welcome.
|
|
16
16
|
|
17
17
|
## <a name="release">Release Notes</a>
|
18
18
|
|
19
|
-
### Release 0.0.
|
19
|
+
### Release 0.0.5
|
20
20
|
|
21
|
-
-
|
21
|
+
- Added tests for WorkQueue.
|
22
22
|
|
23
|
-
- Added
|
23
|
+
- Added busy method to Actor.
|
24
24
|
|
25
25
|
# Plans and Notes
|
26
26
|
|
27
|
-
- WorkQueue test tc_opee_workqueue
|
28
|
-
|
29
27
|
- pick a problem to test against
|
30
28
|
- file density distribution
|
31
29
|
- find all files under a directory
|
@@ -36,18 +34,8 @@ Any comments, thoughts, or suggestions are welcome.
|
|
36
34
|
- wait_finish
|
37
35
|
- ask to print or write report
|
38
36
|
|
39
|
-
- implement Actor max_queue_count limiter
|
40
|
-
- test with actor that reports queue size and pauses for 0.1 seconds
|
41
|
-
- try to send too many requests at it so it has to report busy
|
42
|
-
|
43
|
-
|
44
37
|
- describe patterns for use
|
45
38
|
|
46
|
-
- Is the notion of a job needed to follow processing of an initial input?
|
47
|
-
- avoid using job for storing data though unless rules can be set up to isolate portions of the data to a specific processing path
|
48
|
-
- need something for sharing large chunks of data
|
49
|
-
- maybe just another actor
|
50
|
-
|
51
39
|
### License:
|
52
40
|
|
53
41
|
Copyright (c) 2012, Peter Ohler
|
data/lib/opee.rb
CHANGED
data/lib/opee/actor.rb
CHANGED
@@ -17,7 +17,9 @@ module Opee
|
|
17
17
|
@step_thread = nil
|
18
18
|
@ask_timeout = 0.0
|
19
19
|
@max_queue_count = nil
|
20
|
+
@ask_thread = nil
|
20
21
|
@state = RUNNING
|
22
|
+
@busy = false
|
21
23
|
Env.add_actor(self)
|
22
24
|
set_options(options)
|
23
25
|
@loop = Thread.start(self) do |me|
|
@@ -33,6 +35,7 @@ module Opee
|
|
33
35
|
@ask_mutex.synchronize {
|
34
36
|
a = @queue.pop()
|
35
37
|
}
|
38
|
+
@ask_thread.wakeup() unless @ask_thread.nil?
|
36
39
|
elsif !@idle.empty?
|
37
40
|
@idle_mutex.synchronize {
|
38
41
|
a = @idle.pop()
|
@@ -41,7 +44,9 @@ module Opee
|
|
41
44
|
Env.wake_finish()
|
42
45
|
sleep(1.0)
|
43
46
|
end
|
47
|
+
@busy = true
|
44
48
|
send(a.op, *a.args) unless a.nil?
|
49
|
+
@busy = false
|
45
50
|
if STEP == @state
|
46
51
|
@step_thread.wakeup() unless @step_thread.nil?
|
47
52
|
@state = STOPPED
|
@@ -57,17 +62,28 @@ module Opee
|
|
57
62
|
end
|
58
63
|
|
59
64
|
def set_options(options)
|
60
|
-
|
65
|
+
@max_queue_count = options.fetch(:max_queue_count, @max_queue_count)
|
66
|
+
@ask_timeout = options.fetch(:ask_timeout, @ask_timeout)
|
61
67
|
end
|
62
68
|
|
63
69
|
# deep copy and freeze args if not already frozen or primitive types
|
64
|
-
def
|
70
|
+
def timeout_ask(timeout, op, *args)
|
71
|
+
unless @max_queue_count.nil? || 0 == @max_queue_count || @queue.size() < @max_queue_count
|
72
|
+
@ask_thread = Thread.current
|
73
|
+
sleep(timeout) unless timeout.nil?
|
74
|
+
@ask_thread = nil
|
75
|
+
raise BusyError.new() unless @queue.size() < @max_queue_count
|
76
|
+
end
|
65
77
|
@ask_mutex.synchronize {
|
66
78
|
@queue.insert(0, Act.new(op, args))
|
67
79
|
}
|
68
80
|
@loop.wakeup() if RUNNING == @state
|
69
81
|
end
|
70
82
|
|
83
|
+
def ask(op, *args)
|
84
|
+
timeout_ask(@ask_timeout, op, *args)
|
85
|
+
end
|
86
|
+
|
71
87
|
def on_idle(op, *args)
|
72
88
|
@idle_mutex.synchronize {
|
73
89
|
@idle.insert(0, Act.new(op, args))
|
@@ -91,6 +107,18 @@ module Opee
|
|
91
107
|
@queue.length + @priority.length + @idle.length
|
92
108
|
end
|
93
109
|
|
110
|
+
def busy?
|
111
|
+
@busy || !@queue.empty? || !@priority.empty? || !@idle.empty?
|
112
|
+
end
|
113
|
+
|
114
|
+
def ask_timeout()
|
115
|
+
@ask_timeout
|
116
|
+
end
|
117
|
+
|
118
|
+
def max_queue_count
|
119
|
+
@max_queue_count
|
120
|
+
end
|
121
|
+
|
94
122
|
def stop()
|
95
123
|
@state = STOPPED
|
96
124
|
end
|
@@ -125,6 +153,14 @@ module Opee
|
|
125
153
|
|
126
154
|
private
|
127
155
|
|
156
|
+
def ask_timeout=(timeout)
|
157
|
+
@ask_timeout = timeout
|
158
|
+
end
|
159
|
+
|
160
|
+
def max_queue_count=(max)
|
161
|
+
@max_queue_count = max
|
162
|
+
end
|
163
|
+
|
128
164
|
class Act
|
129
165
|
attr_accessor :op
|
130
166
|
attr_accessor :args
|
data/lib/opee/env.rb
CHANGED
@@ -82,6 +82,11 @@ module Opee
|
|
82
82
|
cnt
|
83
83
|
end
|
84
84
|
|
85
|
+
def self.busy?
|
86
|
+
@@actors.each { |a| return true if a.busy? }
|
87
|
+
false
|
88
|
+
end
|
89
|
+
|
85
90
|
def self.stop()
|
86
91
|
@@actors.each { |a| a.stop() }
|
87
92
|
end
|
@@ -94,7 +99,7 @@ module Opee
|
|
94
99
|
def self.wait_finish()
|
95
100
|
@@finish_thread = Thread.current
|
96
101
|
@@actors.each { |a| a.wakeup() }
|
97
|
-
while
|
102
|
+
while busy?
|
98
103
|
sleep(0.2) # actors should wake up when queue is empty
|
99
104
|
end
|
100
105
|
end
|
data/lib/opee/errors.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
|
2
|
+
module Opee
|
3
|
+
class MissingOptionError < Exception
|
4
|
+
def initialize(option, msg)
|
5
|
+
super("option #{option} for #{msg} missing")
|
6
|
+
end
|
7
|
+
end # MissingOptionError
|
8
|
+
|
9
|
+
class BusyError < Exception
|
10
|
+
def initialize()
|
11
|
+
super("Busy, try again later")
|
12
|
+
end
|
13
|
+
end # BusyError
|
14
|
+
|
15
|
+
end # Opee
|
data/lib/opee/log.rb
CHANGED
@@ -27,6 +27,7 @@ module Opee
|
|
27
27
|
# picked up by the Actor method_missing() method.
|
28
28
|
|
29
29
|
def set_options(options)
|
30
|
+
super(options)
|
30
31
|
if !(filename = options[:filename]).nil?
|
31
32
|
max_file_size = options.fetch(:max_file_size, options.fetch(:shift_size, 1048576))
|
32
33
|
max_file_count = options.fetch(:max_file_count, options.fetch(:shift_age, 7))
|
data/lib/opee/version.rb
CHANGED
data/lib/opee/workqueue.rb
CHANGED
@@ -6,6 +6,9 @@ module Opee
|
|
6
6
|
@workers = []
|
7
7
|
@work_queue = []
|
8
8
|
@method = nil
|
9
|
+
@max_job_count = 0
|
10
|
+
@job_timeout = 3.0
|
11
|
+
@add_thread = nil
|
9
12
|
super(options)
|
10
13
|
end
|
11
14
|
|
@@ -13,10 +16,36 @@ module Opee
|
|
13
16
|
@work_queue.size
|
14
17
|
end
|
15
18
|
|
19
|
+
def worker_count()
|
20
|
+
@workers.size
|
21
|
+
end
|
22
|
+
|
23
|
+
def busy?
|
24
|
+
!@work_queue.empty? || super
|
25
|
+
end
|
26
|
+
|
27
|
+
def ask(op, *args)
|
28
|
+
if :add == op && 0 < @max_job_count && (@work_queue.size() + @queue.size()) >= @max_job_count
|
29
|
+
unless 0.0 >= @job_timeout
|
30
|
+
@add_thread = Thread.current
|
31
|
+
give_up_at = Time.now + @job_timeout
|
32
|
+
until Time.now > give_up_at || (@work_queue.size() + @queue.size()) < @max_job_count
|
33
|
+
sleep(@job_timeout)
|
34
|
+
end
|
35
|
+
@add_thread = nil
|
36
|
+
end
|
37
|
+
raise BusyError.new() unless @work_queue.size() < @max_job_count
|
38
|
+
end
|
39
|
+
super
|
40
|
+
end
|
41
|
+
|
16
42
|
private
|
17
43
|
|
18
44
|
def set_options(options)
|
19
|
-
|
45
|
+
super(options)
|
46
|
+
raise MissingOptionError.new(:method, "for processing jobs") if (@method = options[:method]).nil?
|
47
|
+
@max_job_count = options.fetch(:max_job_count, @max_job_count)
|
48
|
+
@job_timeout = options.fetch(:job_timeout, @job_timeout)
|
20
49
|
end
|
21
50
|
|
22
51
|
def add(job)
|
@@ -33,6 +62,7 @@ module Opee
|
|
33
62
|
@workers.insert(0, worker) unless @workers.include?(worker)
|
34
63
|
else
|
35
64
|
job = @work_queue.pop()
|
65
|
+
@add_thread.wakeup() unless @add_thread.nil?
|
36
66
|
worker.ask(@method, job)
|
37
67
|
end
|
38
68
|
end
|
data/test/relay.rb
CHANGED
@@ -12,6 +12,7 @@ class Relay < ::Opee::Actor
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def set_options(options)
|
15
|
+
super(options)
|
15
16
|
@buddy = options[:buddy]
|
16
17
|
end
|
17
18
|
|
@@ -22,4 +23,10 @@ class Relay < ::Opee::Actor
|
|
22
23
|
@buddy.ask(:relay, data) unless @buddy.nil?
|
23
24
|
end
|
24
25
|
|
26
|
+
def slow(delay)
|
27
|
+
@last_data = [] unless @last_data.is_a?(Array)
|
28
|
+
@last_data << queue_count()
|
29
|
+
sleep(delay)
|
30
|
+
end
|
31
|
+
|
25
32
|
end # Relay
|
data/test/tc_opee_actor.rb
CHANGED
@@ -81,4 +81,11 @@ class OpeeTest < ::Test::Unit::TestCase
|
|
81
81
|
a.close()
|
82
82
|
end
|
83
83
|
|
84
|
+
def test_opee_actor_max_queue_count
|
85
|
+
a = ::Relay.new(:max_queue_count => 4, :ask_timeout => 1.0)
|
86
|
+
10.times { |i| a.ask(:slow, 0.1) }
|
87
|
+
::Opee::Env.wait_close()
|
88
|
+
assert(4 > a.last_data.max)
|
89
|
+
end
|
90
|
+
|
84
91
|
end # OpeeTest
|
@@ -0,0 +1,82 @@
|
|
1
|
+
#!/usr/bin/env ruby -wW2
|
2
|
+
# encoding: UTF-8
|
3
|
+
|
4
|
+
[ File.dirname(__FILE__),
|
5
|
+
File.join(File.dirname(__FILE__), "../lib")
|
6
|
+
].each { |path| $: << path unless $:.include?(path) }
|
7
|
+
|
8
|
+
require 'test/unit'
|
9
|
+
require 'stringio'
|
10
|
+
require 'opee'
|
11
|
+
|
12
|
+
class Worker < ::Opee::Actor
|
13
|
+
|
14
|
+
def initialize(options={})
|
15
|
+
@collector = nil
|
16
|
+
@work_queue = nil
|
17
|
+
super(options)
|
18
|
+
@work_queue.ask(:ready, self)
|
19
|
+
end
|
20
|
+
|
21
|
+
def set_options(options)
|
22
|
+
super(options)
|
23
|
+
@collector = options[:collector]
|
24
|
+
@work_queue = options[:work_queue]
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def do_it(num)
|
30
|
+
@collector.ask(:accept, num * 2)
|
31
|
+
@work_queue.ask(:ready, self)
|
32
|
+
end
|
33
|
+
|
34
|
+
def how_busy(num)
|
35
|
+
@collector.ask(:accept, @work_queue.work_queue_size())
|
36
|
+
sleep(0.1)
|
37
|
+
@work_queue.ask(:ready, self)
|
38
|
+
end
|
39
|
+
|
40
|
+
end # Worker
|
41
|
+
|
42
|
+
class Collector < ::Opee::Actor
|
43
|
+
attr_reader :results
|
44
|
+
|
45
|
+
def initialize(options={})
|
46
|
+
@results = []
|
47
|
+
super(options)
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def accept(num)
|
53
|
+
@results << num
|
54
|
+
end
|
55
|
+
|
56
|
+
end # Collector
|
57
|
+
|
58
|
+
class OpeeTest < ::Test::Unit::TestCase
|
59
|
+
|
60
|
+
def test_opee_workqueue_basic
|
61
|
+
col = Collector.new()
|
62
|
+
wq = ::Opee::WorkQueue.new(:method => :do_it,
|
63
|
+
:max_job_count => 10,
|
64
|
+
:job_timeout => 1.0)
|
65
|
+
4.times { |i| Worker.new(:collector => col, :work_queue => wq) }
|
66
|
+
10.times { |i| wq.ask(:add, i) }
|
67
|
+
::Opee::Env.wait_close()
|
68
|
+
assert_equal([0, 2, 4, 6, 8, 10, 12, 14, 16, 18], col.results.sort())
|
69
|
+
end
|
70
|
+
|
71
|
+
def test_opee_workqueue_busy
|
72
|
+
col = Collector.new()
|
73
|
+
wq = ::Opee::WorkQueue.new(:method => :how_busy,
|
74
|
+
:max_job_count => 4,
|
75
|
+
:job_timeout => 1.0)
|
76
|
+
2.times { |i| Worker.new(:collector => col, :work_queue => wq) }
|
77
|
+
20.times { |i| wq.ask(:add, i) }
|
78
|
+
::Opee::Env.wait_close()
|
79
|
+
assert(4 >= col.results.max)
|
80
|
+
end
|
81
|
+
|
82
|
+
end # OpeeTest
|
data/test/ts_opee.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: opee
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-05-
|
12
|
+
date: 2012-05-03 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: ! 'An experimental Object-base Parallel Evaluation Environment. '
|
15
15
|
email: peter@ohler.com
|
@@ -20,6 +20,7 @@ extra_rdoc_files:
|
|
20
20
|
files:
|
21
21
|
- lib/opee/actor.rb
|
22
22
|
- lib/opee/env.rb
|
23
|
+
- lib/opee/errors.rb
|
23
24
|
- lib/opee/log.rb
|
24
25
|
- lib/opee/version.rb
|
25
26
|
- lib/opee/workqueue.rb
|
@@ -28,6 +29,7 @@ files:
|
|
28
29
|
- test/tc_opee_actor.rb
|
29
30
|
- test/tc_opee_env.rb
|
30
31
|
- test/tc_opee_log.rb
|
32
|
+
- test/tc_opee_workqueue.rb
|
31
33
|
- test/tests.rb
|
32
34
|
- test/ts_opee.rb
|
33
35
|
- LICENSE
|
@@ -54,9 +56,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
54
56
|
version: '0'
|
55
57
|
requirements: []
|
56
58
|
rubyforge_project: opee
|
57
|
-
rubygems_version: 1.8.
|
59
|
+
rubygems_version: 1.8.23
|
58
60
|
signing_key:
|
59
61
|
specification_version: 3
|
60
62
|
summary: An experimental Object-base Parallel Evaluation Environment.
|
61
63
|
test_files: []
|
62
|
-
has_rdoc: true
|