concur 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/completer.rb +12 -0
- data/lib/executor.rb +33 -1
- data/lib/executors/event_machine_executor.rb +20 -7
- data/lib/future.rb +12 -4
- data/lib/futures/event_machine_future.rb +137 -11
- metadata +28 -9
- data/test/executor_spec.rb +0 -87
- data/test/job.rb +0 -51
data/lib/completer.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# Decouples execution of tasks from getting completed tasks so you can add tasks for execution to a Completer
|
2
|
+
# and retrieve completed tasks (first completed, first out) from it.
|
3
|
+
#
|
4
|
+
# based on http://download.oracle.com/javase/6/docs/api/java/util/concurrent/CompletionService.html
|
5
|
+
|
6
|
+
module Concur
|
7
|
+
|
8
|
+
class Completer
|
9
|
+
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
data/lib/executor.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
+
require 'faraday'
|
1
2
|
require_relative 'runnable'
|
2
3
|
require_relative 'future'
|
3
4
|
require_relative 'thread_pool'
|
4
5
|
|
5
|
-
|
6
6
|
module Concur
|
7
7
|
|
8
8
|
|
@@ -59,8 +59,40 @@ module Concur
|
|
59
59
|
def shutdown
|
60
60
|
@thread_pool.shutdown
|
61
61
|
end
|
62
|
+
|
63
|
+
def http_request(params, &blk)
|
64
|
+
|
65
|
+
f = StandardFuture.new do
|
66
|
+
conn = Faraday.new(:url => params[:base_url]) do |builder|
|
67
|
+
# builder.use Faraday::Request::UrlEncoded # convert request params as "www-form-urlencoded"
|
68
|
+
# builder.use Faraday::Request::JSON # encode request params as json
|
69
|
+
# builder.use Faraday::Response::Logger # log the request to STDOUT
|
70
|
+
builder.use Faraday::Adapter::NetHttp # make http requests with Net::HTTP
|
71
|
+
#
|
72
|
+
# # or, use shortcuts:
|
73
|
+
# builder.request :url_encoded
|
74
|
+
# builder.request :json
|
75
|
+
# builder.response :logger
|
76
|
+
# builder.adapter :net_http
|
77
|
+
end
|
78
|
+
if params[:http_method] == :post
|
79
|
+
response = conn.post params[:path]
|
80
|
+
else
|
81
|
+
response = conn.get params[:path]
|
82
|
+
end
|
83
|
+
if block_given?
|
84
|
+
response = blk.call(response)
|
85
|
+
end
|
86
|
+
response
|
87
|
+
end
|
88
|
+
@thread_pool.process(f)
|
89
|
+
f
|
90
|
+
end
|
91
|
+
|
92
|
+
|
62
93
|
end
|
63
94
|
|
95
|
+
|
64
96
|
# todo: should maybe have these backends extend Executor and just override what's necessary
|
65
97
|
class SingleThreaded
|
66
98
|
def process(f)
|
@@ -4,17 +4,22 @@ require_relative '../futures/event_machine_future'
|
|
4
4
|
module Concur
|
5
5
|
class EventMachineExecutor
|
6
6
|
|
7
|
-
|
8
7
|
def initialize
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
8
|
+
unless EventMachine.reactor_running? # also check EventMachine.reactor_thread? ??
|
9
|
+
@in_thread = true
|
10
|
+
@em_thread = Thread.new do
|
11
|
+
EventMachine.run do
|
12
|
+
puts 'Starting EventMachineExecutor...'
|
13
13
|
# @futures.each do |f|
|
14
14
|
#
|
15
15
|
# end
|
16
|
+
end
|
17
|
+
puts 'EventMachine loop done in executor thread'
|
16
18
|
end
|
17
|
-
|
19
|
+
sleep 0.1 # let EM startup
|
20
|
+
else
|
21
|
+
puts 'Reactor already running...'
|
22
|
+
@in_thread = false
|
18
23
|
end
|
19
24
|
end
|
20
25
|
|
@@ -26,9 +31,17 @@ module Concur
|
|
26
31
|
end
|
27
32
|
|
28
33
|
def shutdown
|
29
|
-
@
|
34
|
+
if @in_thread
|
35
|
+
EventMachine.stop
|
36
|
+
@em_thread.kill
|
37
|
+
end
|
30
38
|
puts 'shutdown'
|
31
39
|
end
|
32
40
|
|
41
|
+
# Abstracts the http client used that is suitable for the executor
|
42
|
+
def http_request(params, &blk)
|
43
|
+
f = EventMachineFuture2.new(params, &blk)
|
44
|
+
end
|
45
|
+
|
33
46
|
end
|
34
47
|
end
|
data/lib/future.rb
CHANGED
@@ -28,15 +28,19 @@ module Concur
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def run
|
31
|
+
puts 'running StandardFuture'
|
31
32
|
begin
|
32
33
|
@result = @callable.call
|
34
|
+
puts 'callable called: ' + @result.inspect
|
33
35
|
rescue Exception => ex
|
36
|
+
puts 'error occurred'
|
34
37
|
@ex = ex
|
35
38
|
end
|
36
39
|
@mutex.synchronize do # do we even need to synchronize? run should only ever be called once
|
37
40
|
@complete = true
|
38
|
-
@cv.broadcast
|
39
41
|
end
|
42
|
+
@cv.broadcast
|
43
|
+
|
40
44
|
end
|
41
45
|
|
42
46
|
def call
|
@@ -50,9 +54,11 @@ module Concur
|
|
50
54
|
# Returns results. Will wait for thread to complete execution if not already complete.
|
51
55
|
def get
|
52
56
|
# @thread.value
|
53
|
-
@
|
54
|
-
|
55
|
-
@
|
57
|
+
unless @complete
|
58
|
+
@mutex.synchronize do
|
59
|
+
unless @complete
|
60
|
+
@cv.wait(@mutex)
|
61
|
+
end
|
56
62
|
end
|
57
63
|
end
|
58
64
|
if @ex
|
@@ -61,4 +67,6 @@ module Concur
|
|
61
67
|
@result
|
62
68
|
end
|
63
69
|
end
|
70
|
+
|
71
|
+
|
64
72
|
end
|
@@ -43,13 +43,14 @@ module Concur
|
|
43
43
|
|
44
44
|
|
45
45
|
end
|
46
|
+
|
46
47
|
def errback &blk
|
47
|
-
|
48
|
-
|
48
|
+
@errblk = blk
|
49
|
+
end
|
49
50
|
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
def callback &blk
|
52
|
+
@callblk = blk
|
53
|
+
end
|
53
54
|
|
54
55
|
def errback2 &blk
|
55
56
|
puts 'EventMachineFutureCallback.errback'
|
@@ -69,7 +70,7 @@ module Concur
|
|
69
70
|
if @callblk
|
70
71
|
@result = @callblk.call(@callbackable)
|
71
72
|
end
|
72
|
-
blk.call(@result)
|
73
|
+
blk.call(@result)
|
73
74
|
end
|
74
75
|
@callbackable.callback &proc
|
75
76
|
end
|
@@ -86,10 +87,134 @@ module Concur
|
|
86
87
|
# end
|
87
88
|
end
|
88
89
|
|
90
|
+
class HttpResponseWrapper
|
91
|
+
def initialize(em_http)
|
92
|
+
@em_http = em_http
|
93
|
+
end
|
94
|
+
|
95
|
+
def headers
|
96
|
+
@em_http.response_header
|
97
|
+
end
|
98
|
+
|
99
|
+
def body
|
100
|
+
@em_http.response
|
101
|
+
end
|
102
|
+
|
103
|
+
def status
|
104
|
+
@em_http.response_header.status
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
108
|
+
|
109
|
+
class EventMachineFuture2
|
110
|
+
require 'em-http'
|
111
|
+
include Concur::Future
|
112
|
+
|
113
|
+
attr_accessor :ex, :result
|
114
|
+
|
115
|
+
def initialize(http_data, &block)
|
116
|
+
@http_data = http_data
|
117
|
+
if block_given?
|
118
|
+
@blk = block
|
119
|
+
end
|
120
|
+
|
121
|
+
puts 'http_data=' + http_data.inspect
|
122
|
+
|
123
|
+
req = EventMachine::HttpRequest.new(http_data[:base_url])
|
124
|
+
|
125
|
+
opts = {:timeout => http_data[:timeout], :head => http_data[:headers]} #, :ssl => true
|
126
|
+
|
127
|
+
if http_data[:http_method] == :post
|
128
|
+
http = req.post opts.merge(:path=>http_data[:path], :body=>http_data[:body])
|
129
|
+
elsif http_data[:http_method] == :put
|
130
|
+
http = req.put opts.merge(:path=>http_data[:path], :body=>http_data[:body])
|
131
|
+
elsif http_data[:http_method] == :head
|
132
|
+
http = req.head opts.merge(:path=>http_data[:path])
|
133
|
+
elsif http_data[:http_method] == :delete
|
134
|
+
http = req.delete opts.merge(:path=>http_data[:path])
|
135
|
+
else
|
136
|
+
http = req.get opts.merge(:path=>http_data[:path], :query=>http_data[:query])
|
137
|
+
end
|
138
|
+
|
139
|
+
if http.error.empty?
|
140
|
+
http.errback {
|
141
|
+
begin
|
142
|
+
puts 'Uh oh'
|
143
|
+
p http.response_header.status
|
144
|
+
p http.response_header
|
145
|
+
p http.response
|
146
|
+
@ex = StandardError.new("ERROR: #{http.response_header.status} #{http.response}")
|
147
|
+
rescue => ex
|
148
|
+
@ex = ex
|
149
|
+
end
|
150
|
+
self.complete
|
151
|
+
}
|
152
|
+
http.callback {
|
153
|
+
begin
|
154
|
+
puts 'success callback'
|
155
|
+
# puts 'status=' + http.response_header.status
|
156
|
+
# puts 'response header=' + http.response_header
|
157
|
+
if @blk
|
158
|
+
@result = @blk.call(HttpResponseWrapper.new(http))
|
159
|
+
else
|
160
|
+
@result = HttpResponseWrapper.new(http)
|
161
|
+
end
|
162
|
+
rescue => ex
|
163
|
+
@ex = ex
|
164
|
+
end
|
165
|
+
self.complete
|
166
|
+
}
|
167
|
+
else
|
168
|
+
p http.error.inspect
|
169
|
+
@ex = StandardError.new(http.error)
|
170
|
+
self.complete
|
171
|
+
end
|
172
|
+
|
173
|
+
|
174
|
+
end
|
175
|
+
|
176
|
+
|
177
|
+
def complete
|
178
|
+
@complete = true
|
179
|
+
end
|
180
|
+
|
181
|
+
def complete?
|
182
|
+
@complete
|
183
|
+
end
|
184
|
+
|
185
|
+
# Returns results. Will wait for thread to complete execution if not already complete.
|
186
|
+
def get
|
187
|
+
# @thread.value
|
188
|
+
# if !@complete
|
189
|
+
# # todo: gotta be a better way
|
190
|
+
## puts 'sleeping'
|
191
|
+
## sleep 0.1
|
192
|
+
# @fiber = Fiber.new do
|
193
|
+
# while !@complete
|
194
|
+
# sleep 0.1
|
195
|
+
# end
|
196
|
+
# Fiber.yield # give back control
|
197
|
+
# end
|
198
|
+
# @fiber.resume # start fiber
|
199
|
+
# end
|
200
|
+
while !@complete
|
201
|
+
sleep 0.1
|
202
|
+
end
|
203
|
+
return get_response
|
204
|
+
end
|
205
|
+
|
206
|
+
def get_response
|
207
|
+
if @ex
|
208
|
+
raise @ex
|
209
|
+
end
|
210
|
+
@result
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
89
214
|
class EventMachineFuture
|
90
215
|
include Concur::Future
|
91
216
|
|
92
|
-
attr_accessor :ex
|
217
|
+
attr_accessor :ex, :result
|
93
218
|
|
94
219
|
def initialize(callable, &block)
|
95
220
|
|
@@ -101,12 +226,11 @@ module Concur
|
|
101
226
|
end
|
102
227
|
end
|
103
228
|
|
104
|
-
|
105
229
|
def run
|
106
230
|
puts 'EMFuture.run'
|
107
231
|
p @callable
|
108
232
|
begin
|
109
|
-
@callbackable = @callable.call
|
233
|
+
@callbackable = @callable.call(self)
|
110
234
|
puts 'done @callable.call ' + @callbackable.inspect
|
111
235
|
rescue Exception => ex
|
112
236
|
@ex = ex
|
@@ -122,7 +246,8 @@ module Concur
|
|
122
246
|
@ex = EventMachineError.new(http)
|
123
247
|
complete
|
124
248
|
}
|
125
|
-
@result = (http.callback2 {|result|
|
249
|
+
@result = (http.callback2 { |result|
|
250
|
+
puts 'completion errback'
|
126
251
|
@result = result
|
127
252
|
complete
|
128
253
|
})
|
@@ -146,7 +271,7 @@ module Concur
|
|
146
271
|
# Returns results. Will wait for thread to complete execution if not already complete.
|
147
272
|
def get
|
148
273
|
# @thread.value
|
149
|
-
while
|
274
|
+
while !@complete
|
150
275
|
# todo: gotta be a better way
|
151
276
|
puts 'sleeping'
|
152
277
|
sleep 0.5
|
@@ -161,4 +286,5 @@ module Concur
|
|
161
286
|
@result
|
162
287
|
end
|
163
288
|
end
|
289
|
+
|
164
290
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: concur
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.7
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Travis Reeder
|
@@ -10,9 +10,30 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-05-
|
14
|
-
dependencies:
|
15
|
-
|
13
|
+
date: 2011-05-23 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: eventmachine
|
17
|
+
prerelease: false
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: faraday
|
28
|
+
prerelease: false
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "0"
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id002
|
16
37
|
description: A concurrency library for Ruby inspired by java.util.concurrency. By http://www.appoxy.com
|
17
38
|
email: travis@appoxy.com
|
18
39
|
executables: []
|
@@ -22,6 +43,7 @@ extensions: []
|
|
22
43
|
extra_rdoc_files:
|
23
44
|
- README.markdown
|
24
45
|
files:
|
46
|
+
- lib/completer.rb
|
25
47
|
- lib/concur.rb
|
26
48
|
- lib/executor.rb
|
27
49
|
- lib/executors/event_machine_executor.rb
|
@@ -31,8 +53,6 @@ files:
|
|
31
53
|
- lib/runnable.rb
|
32
54
|
- lib/thread_pool.rb
|
33
55
|
- README.markdown
|
34
|
-
- test/executor_spec.rb
|
35
|
-
- test/job.rb
|
36
56
|
homepage: http://github.com/appoxy/concur/
|
37
57
|
licenses: []
|
38
58
|
|
@@ -60,6 +80,5 @@ rubygems_version: 1.7.2
|
|
60
80
|
signing_key:
|
61
81
|
specification_version: 3
|
62
82
|
summary: A concurrency library for Ruby inspired by java.util.concurrency. By http://www.appoxy.com
|
63
|
-
test_files:
|
64
|
-
|
65
|
-
- test/job.rb
|
83
|
+
test_files: []
|
84
|
+
|
data/test/executor_spec.rb
DELETED
@@ -1,87 +0,0 @@
|
|
1
|
-
require 'rspec'
|
2
|
-
require 'neverblock'
|
3
|
-
|
4
|
-
require_relative '../lib/concur'
|
5
|
-
require_relative 'job'
|
6
|
-
|
7
|
-
@@durations = []
|
8
|
-
|
9
|
-
def run_jobs(name, executor, times, options={})
|
10
|
-
puts "Running #{name}..."
|
11
|
-
start_time = Time.now
|
12
|
-
|
13
|
-
jobs = []
|
14
|
-
times.times do |i|
|
15
|
-
job = Job.new(i, options)
|
16
|
-
jobs << executor.execute(job)
|
17
|
-
end
|
18
|
-
jobs.each do |j|
|
19
|
-
puts "result=#{j.get}"
|
20
|
-
end
|
21
|
-
concurrent_duration = Time.now - start_time
|
22
|
-
o = "#{name} duration=" + concurrent_duration.to_s
|
23
|
-
puts o
|
24
|
-
@@durations << o
|
25
|
-
concurrent_duration
|
26
|
-
end
|
27
|
-
|
28
|
-
describe Concur::Executor do
|
29
|
-
describe "#score" do
|
30
|
-
it "runs faster in parallel" do
|
31
|
-
times = 10
|
32
|
-
|
33
|
-
job = Job.new(1)
|
34
|
-
puts 'runnable? ' + job.is_a?(Concur::Runnable).to_s
|
35
|
-
non_concurrent_duration = 0
|
36
|
-
#
|
37
|
-
executor = Concur::Executor.new_single_threaded_executor
|
38
|
-
non_concurrent_duration =run_jobs("non concurrent", executor, times)
|
39
|
-
executor.shutdown
|
40
|
-
|
41
|
-
executor = Concur::Executor.new_multi_threaded_executor
|
42
|
-
concurrent_duration = run_jobs("multi thread", executor, times)
|
43
|
-
concurrent_duration.should be < (non_concurrent_duration/2)
|
44
|
-
executor.shutdown
|
45
|
-
|
46
|
-
executor = Concur::Executor.new_thread_pool_executor(10)
|
47
|
-
pooled_duration = run_jobs("thread pool", executor, times)
|
48
|
-
pooled_duration.should be < (non_concurrent_duration/2)
|
49
|
-
executor.shutdown
|
50
|
-
|
51
|
-
# Don't think I know how to use NeverBlock properly
|
52
|
-
# executor = Concur::Executor.new_neverblock_executor(10)
|
53
|
-
# neverblock_duration = run_jobs("never blocked", executor, times)
|
54
|
-
# neverblock_duration.should be < (non_concurrent_duration/2)
|
55
|
-
# executor.shutdown
|
56
|
-
|
57
|
-
executor = Concur::Executor.new_eventmachine_executor()
|
58
|
-
em_duration = run_jobs("eventmachine", executor, times, :em=>true)
|
59
|
-
em_duration.should be < (non_concurrent_duration/2)
|
60
|
-
executor.shutdown
|
61
|
-
|
62
|
-
@@durations.each do |s|
|
63
|
-
puts s
|
64
|
-
end
|
65
|
-
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
describe Concur::Future do
|
71
|
-
|
72
|
-
describe "#new" do
|
73
|
-
|
74
|
-
it "can accept blocks" do
|
75
|
-
future = Concur::Future.new do
|
76
|
-
puts "i'm in the block"
|
77
|
-
"result of block"
|
78
|
-
end
|
79
|
-
puts 'get=' + future.get
|
80
|
-
future.get.should == "result of block"
|
81
|
-
end
|
82
|
-
|
83
|
-
end
|
84
|
-
|
85
|
-
end
|
86
|
-
|
87
|
-
|
data/test/job.rb
DELETED
@@ -1,51 +0,0 @@
|
|
1
|
-
require_relative '../lib/runnable'
|
2
|
-
require 'rest-client'
|
3
|
-
require 'eventmachine'
|
4
|
-
|
5
|
-
class Job
|
6
|
-
include Concur::Runnable
|
7
|
-
|
8
|
-
def initialize(i, options={})
|
9
|
-
@i = i
|
10
|
-
if options[:em]
|
11
|
-
@em = true
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def em_request(url)
|
16
|
-
puts 'emrequest for ' + url
|
17
|
-
http = EventMachine::Protocols::HttpClient.request(
|
18
|
-
:host => url,
|
19
|
-
:port => 80,
|
20
|
-
:request => "/"
|
21
|
-
)
|
22
|
-
http.callback { |response|
|
23
|
-
puts response[:status]
|
24
|
-
puts response[:headers]
|
25
|
-
puts response[:content]
|
26
|
-
}
|
27
|
-
end
|
28
|
-
|
29
|
-
def run
|
30
|
-
puts "Starting #{@i}... em? #{@em}"
|
31
|
-
# sleep 1
|
32
|
-
urls = ["www.yahoo.com", "www.microsoft.com"] # , "www.github.com"
|
33
|
-
if @em
|
34
|
-
urls.each do |u|
|
35
|
-
em_request(u)
|
36
|
-
end
|
37
|
-
else
|
38
|
-
urls.each do |u|
|
39
|
-
get(u)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
puts "Finished #{@i}"
|
43
|
-
"response #{@i}"
|
44
|
-
end
|
45
|
-
|
46
|
-
def get(url)
|
47
|
-
puts 'getting ' + url
|
48
|
-
RestClient.get "http://#{url}"
|
49
|
-
end
|
50
|
-
|
51
|
-
end
|