simple_worker 0.0.13 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/http_enabled.rb +4 -3
- data/lib/simple_worker.rb +35 -10
- data/lib/simple_worker_error.rb +22 -0
- data/lib/worker.rb +5 -0
- data/test/scheduled_worker.rb +9 -0
- data/test/simple_worker_tests.rb +75 -46
- data/test/{test_runner.rb → test_worker.rb} +5 -6
- metadata +5 -3
data/lib/http_enabled.rb
CHANGED
@@ -3,6 +3,7 @@ require 'active_support'
|
|
3
3
|
require 'net/http'
|
4
4
|
require 'base64'
|
5
5
|
|
6
|
+
require File.join(File.dirname(__FILE__), 'simple_worker_error')
|
6
7
|
|
7
8
|
begin
|
8
9
|
require 'digest/hmac'
|
@@ -86,11 +87,11 @@ module SimpleWorker
|
|
86
87
|
when Net::HTTPSuccess
|
87
88
|
# puts 'response body=' + res.body
|
88
89
|
ret = res.body
|
90
|
+
when Net::HTTPClientError
|
91
|
+
raise SimpleWorker::ClientError.new(res.class.name, ActiveSupport::JSON.decode(res.body))
|
89
92
|
else
|
90
93
|
#res.error
|
91
|
-
|
92
|
-
puts res.body
|
93
|
-
ret = res.body
|
94
|
+
raise SimpleWorker::ServiceError.new(res.class.name, res.body)
|
94
95
|
end
|
95
96
|
return ret
|
96
97
|
end
|
data/lib/simple_worker.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
require File.join(File.dirname(__FILE__), 'http_enabled')
|
2
2
|
require File.join(File.dirname(__FILE__), 'api_auth')
|
3
3
|
require File.join(File.dirname(__FILE__), 'worker')
|
4
|
+
require File.join(File.dirname(__FILE__), 'simple_worker_error')
|
5
|
+
|
4
6
|
|
5
7
|
module SimpleWorker
|
6
8
|
|
@@ -9,6 +11,7 @@ module SimpleWorker
|
|
9
11
|
include SimpleWorker::HttpEnabled
|
10
12
|
|
11
13
|
def initialize(access_key, secret_key, options={})
|
14
|
+
puts 'Starting SimpleWorker::Service...'
|
12
15
|
super(access_key, secret_key, options)
|
13
16
|
end
|
14
17
|
|
@@ -22,7 +25,7 @@ module SimpleWorker
|
|
22
25
|
options = {"code"=>mystring, "class_name"=>class_name}
|
23
26
|
response = run_http(@host, @access_key, @secret_key, :post, "code/put", nil, options)
|
24
27
|
# puts "response=" + response
|
25
|
-
|
28
|
+
parse_response response
|
26
29
|
end
|
27
30
|
|
28
31
|
#
|
@@ -36,10 +39,11 @@ module SimpleWorker
|
|
36
39
|
hash_to_send = {}
|
37
40
|
hash_to_send["data"] = data
|
38
41
|
hash_to_send["class_name"] = class_name
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
42
|
+
if defined?(RAILS_ENV)
|
43
|
+
hash_to_send["rails_env"] = RAILS_ENV
|
44
|
+
end
|
45
|
+
# puts 'hash_to_send=' + hash_to_send.inspect
|
46
|
+
return queue_raw(class_name, hash_to_send)
|
43
47
|
|
44
48
|
end
|
45
49
|
|
@@ -49,7 +53,18 @@ module SimpleWorker
|
|
49
53
|
# puts 'hash_to_send=' + hash_to_send.inspect
|
50
54
|
response = run_http(@host, @access_key, @secret_key, :put, "queue/add", hash_to_send, params)
|
51
55
|
# puts "response=" + response
|
52
|
-
|
56
|
+
|
57
|
+
parse_response response
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
def parse_response(response)
|
62
|
+
begin
|
63
|
+
return ActiveSupport::JSON.decode(response)
|
64
|
+
rescue => ex
|
65
|
+
puts 'response that caused error = ' + response.to_s
|
66
|
+
raise ex
|
67
|
+
end
|
53
68
|
end
|
54
69
|
|
55
70
|
|
@@ -76,7 +91,7 @@ module SimpleWorker
|
|
76
91
|
# puts 'hash_to_send=' + hash_to_send.inspect
|
77
92
|
response = run_http(@host, @access_key, @secret_key, :put, "queue/schedule", hash_to_send, params)
|
78
93
|
# puts "response=" + response
|
79
|
-
|
94
|
+
parse_response response
|
80
95
|
end
|
81
96
|
|
82
97
|
#
|
@@ -93,15 +108,25 @@ module SimpleWorker
|
|
93
108
|
# puts 'hash_to_send=' + hash_to_send.inspect
|
94
109
|
response = run_http(@host, @access_key, @secret_key, :put, "queue/add", hash_to_send, params)
|
95
110
|
# puts "response=" + response
|
96
|
-
|
111
|
+
parse_response response
|
97
112
|
end
|
98
113
|
|
99
114
|
def status(task_id)
|
100
115
|
data = {"task_id"=>task_id}
|
101
116
|
#puts run_http(@access_key, @secret_key, :post, "queue/status", nil, {"task_id"=>@task_id})
|
102
|
-
response = run_http(@host, @access_key, @secret_key, :get, "
|
117
|
+
response = run_http(@host, @access_key, @secret_key, :get, "task/status", nil, data)
|
118
|
+
# puts "response=" + response
|
119
|
+
parse_response response
|
120
|
+
end
|
121
|
+
|
122
|
+
def log(task_id)
|
123
|
+
data = {"task_id"=>task_id}
|
124
|
+
#puts run_http(@access_key, @secret_key, :post, "queue/status", nil, {"task_id"=>@task_id})
|
125
|
+
response = run_http(@host, @access_key, @secret_key, :get, "task/log", nil, data)
|
103
126
|
# puts "response=" + response
|
104
|
-
|
127
|
+
ret = parse_response response
|
128
|
+
ret["log"] = Base64.decode64(ret["log"])
|
129
|
+
ret
|
105
130
|
end
|
106
131
|
|
107
132
|
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module SimpleWorker
|
2
|
+
class ClientError < StandardError
|
3
|
+
|
4
|
+
attr_reader :response_hash
|
5
|
+
|
6
|
+
def initialize(class_name, response_hash)
|
7
|
+
puts 'response-hash=' + response_hash.inspect
|
8
|
+
super("#{class_name} - #{response_hash["msg"]}")
|
9
|
+
@response_hash = response_hash
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class ServiceError < StandardError
|
14
|
+
attr_reader :body
|
15
|
+
|
16
|
+
def initialize(class_name, body)
|
17
|
+
super("#{class_name}")
|
18
|
+
@body = body
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/worker.rb
CHANGED
data/test/simple_worker_tests.rb
CHANGED
@@ -1,46 +1,75 @@
|
|
1
|
-
require 'test/unit'
|
2
|
-
require 'yaml'
|
3
|
-
require File.join(File.dirname(__FILE__), '../lib/simple_worker')
|
4
|
-
|
5
|
-
class SimpleWorkerTests < Test::Unit::TestCase
|
6
|
-
|
7
|
-
def setup
|
8
|
-
@config = YAML::load(File.open(File.expand_path("~/.test-configs/simple_worker.yml")))
|
9
|
-
#puts @config.inspect
|
10
|
-
@access_key = @config['simple_worker']['access_key']
|
11
|
-
@secret_key = @config['simple_worker']['secret_key']
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
# task
|
29
|
-
|
30
|
-
|
31
|
-
#
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
1
|
+
require 'test/unit'
|
2
|
+
require 'yaml'
|
3
|
+
require File.join(File.dirname(__FILE__), '../lib/simple_worker')
|
4
|
+
|
5
|
+
class SimpleWorkerTests < Test::Unit::TestCase
|
6
|
+
|
7
|
+
def setup
|
8
|
+
@config = YAML::load(File.open(File.expand_path("~/.test-configs/simple_worker.yml")))
|
9
|
+
#puts @config.inspect
|
10
|
+
@access_key = @config['simple_worker']['access_key']
|
11
|
+
@secret_key = @config['simple_worker']['secret_key']
|
12
|
+
|
13
|
+
@worker = SimpleWorker::Service.new(@access_key, @secret_key)
|
14
|
+
@worker.host = "http://localhost:3000/api/"
|
15
|
+
end
|
16
|
+
|
17
|
+
def teardown
|
18
|
+
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_queue
|
22
|
+
|
23
|
+
|
24
|
+
# Upload latest runner code
|
25
|
+
@worker.upload(File.join(File.dirname(__FILE__), "./test_runner.rb"), "TestWorker")
|
26
|
+
|
27
|
+
# Add something to queue, get task ID back
|
28
|
+
# Single task
|
29
|
+
response_hash_single = @worker.queue("TestRunner", {"s3_key"=>"single runner", "times"=>10})
|
30
|
+
|
31
|
+
# task set
|
32
|
+
response_hash = @worker.queue("TestRunner", [{"id"=>"local_id", "s3_key"=>"some key", "times"=>4}, {"s3_key"=>"some key2", "times"=>3}, {"s3_key"=>"some key", "times"=>2}])
|
33
|
+
#
|
34
|
+
# Check status
|
35
|
+
tasks = response_hash["tasks"]
|
36
|
+
while tasks.size > 0
|
37
|
+
tasks.each do |t|
|
38
|
+
status_response = @worker.status(t["task_id"])
|
39
|
+
puts 'status for ' + t["task_id"] + ' = ' + status_response["status"]
|
40
|
+
if status_response["status"] == "complete" || status_response["status"] == "error" || status_response["status"] == "cancelled"
|
41
|
+
tasks.delete(t)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# lets try to get the log now too
|
47
|
+
task_id = response_hash_single["tasks"][0]["task_id"]
|
48
|
+
puts 'task_id=' + task_id
|
49
|
+
puts 'log=' + @worker.log(task_id).inspect
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_scheduled
|
54
|
+
|
55
|
+
# Upload latest runner code
|
56
|
+
@worker.upload(File.join(File.dirname(__FILE__), "./scheduled_runner.rb"), "ScheduledWorker")
|
57
|
+
|
58
|
+
start_at = 10.seconds.since
|
59
|
+
#start_at = start_at.gmtime # testing different timezone
|
60
|
+
puts 'start_at =' + start_at.inspect
|
61
|
+
response_hash = @worker.schedule("TestRunner", {"msg"=>"One time test."}, {:start_at=>start_at})
|
62
|
+
puts 'response_hash=' + response_hash.inspect
|
63
|
+
|
64
|
+
start_at = 10.seconds.since
|
65
|
+
response_hash = @worker.schedule("TestRunner", {"msg"=>"Run times test"}, {:start_at=>start_at, :run_every=>30, :run_times=>3})
|
66
|
+
puts 'response_hash=' + response_hash.inspect
|
67
|
+
|
68
|
+
start_at = 10.seconds.since
|
69
|
+
end_at = 2.minutes.since
|
70
|
+
response_hash = @worker.schedule("TestRunner", {"msg"=>"End at test"}, {:start_at=>start_at, :run_every=>30, :end_at=>end_at, :run_times=>20})
|
71
|
+
puts 'response_hash=' + response_hash.inspect
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
@@ -1,19 +1,20 @@
|
|
1
1
|
require 'simple_worker'
|
2
2
|
|
3
|
-
|
3
|
+
# Bump for new checksum.
|
4
|
+
class TestWorker < SimpleWorker::Base
|
4
5
|
|
5
6
|
def run(data=nil)
|
6
|
-
|
7
|
+
log 'running the runner for leroy '.upcase + ' with data: ' + data.inspect
|
7
8
|
@times = data["times"].to_i
|
8
9
|
@times.times do |i|
|
9
|
-
|
10
|
+
log 'running at ' + i.to_s
|
10
11
|
sleep 1
|
11
12
|
set_progress(:percent=> (i / @times * 100))
|
12
13
|
end
|
13
14
|
end
|
14
15
|
|
15
16
|
def set_complete(params=nil)
|
16
|
-
|
17
|
+
log 'SET COMPLETE YAY!' + params[:task_set_id]
|
17
18
|
end
|
18
19
|
|
19
20
|
def progress
|
@@ -23,7 +24,5 @@ class TestRunner < SimpleWorker::Base
|
|
23
24
|
return 0.0
|
24
25
|
end
|
25
26
|
|
26
|
-
|
27
|
-
|
28
27
|
end
|
29
28
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: simple_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Travis Reeder
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
12
|
+
date: 2009-12-13 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -25,6 +25,7 @@ files:
|
|
25
25
|
- lib/api_auth.rb
|
26
26
|
- lib/http_enabled.rb
|
27
27
|
- lib/simple_worker.rb
|
28
|
+
- lib/simple_worker_error.rb
|
28
29
|
- lib/worker.rb
|
29
30
|
- README.markdown
|
30
31
|
has_rdoc: true
|
@@ -56,5 +57,6 @@ signing_key:
|
|
56
57
|
specification_version: 3
|
57
58
|
summary: Classified
|
58
59
|
test_files:
|
60
|
+
- test/scheduled_worker.rb
|
59
61
|
- test/simple_worker_tests.rb
|
60
|
-
- test/
|
62
|
+
- test/test_worker.rb
|