simple_worker 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +13 -0
- data/lib/generators/simple_worker/simple_worker_generator.rb +13 -0
- data/lib/simple_worker/api.rb +10 -8
- data/lib/simple_worker/base.rb +1 -13
- data/lib/{rails2_init.rb → simple_worker/rails2_init.rb} +0 -0
- data/lib/{railtie.rb → simple_worker/railtie.rb} +0 -0
- data/lib/simple_worker/server/overrides.rb +3 -6
- data/lib/simple_worker/server/runner.rb +18 -4
- data/lib/simple_worker/service.rb +48 -13
- data/lib/simple_worker.rb +2 -2
- metadata +6 -5
data/README.markdown
CHANGED
@@ -28,6 +28,19 @@ You really just need your access keys.
|
|
28
28
|
config.access_key = ACCESS_KEY
|
29
29
|
config.secret_key = SECRET_KEY
|
30
30
|
end
|
31
|
+
|
32
|
+
Generate a Worker with Rails
|
33
|
+
----------------------------
|
34
|
+
If you're rolling on Rails, use the following:
|
35
|
+
|
36
|
+
rails generate simple_worker worker_class_in_underscore_format
|
37
|
+
|
38
|
+
This will set you up with a worker in ```app/workers```
|
39
|
+
However, this is not autorequired, so a line similar to this will be required in your initializer:
|
40
|
+
|
41
|
+
Dir.glob("#{Rails.root}/app/workers/*.rb").each{|w| require w}
|
42
|
+
|
43
|
+
This one-liner requires all ```.rb``` files in ```app/workers```
|
31
44
|
|
32
45
|
Write a Worker
|
33
46
|
--------------
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class SimpleWorkerGenerator < Rails::Generators::NamedBase
|
2
|
+
source_root File.expand_path("../templates", __FILE__)
|
3
|
+
|
4
|
+
desc "Creates a new skeleton worker - NAME is camelized"
|
5
|
+
def create_worker_file
|
6
|
+
# file_name needs to be classified
|
7
|
+
@camel = file_name.camelize
|
8
|
+
if not File.directory? "#{Rails.root}/app/workers"
|
9
|
+
Dir.mkdir "#{Rails.root}/app/workers"
|
10
|
+
end
|
11
|
+
template "template_worker.erb", "app/workers/#{file_name}.rb"
|
12
|
+
end
|
13
|
+
end
|
data/lib/simple_worker/api.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'rest_client'
|
2
|
+
|
1
3
|
module SimpleWorker
|
2
4
|
module Api
|
3
5
|
|
@@ -45,9 +47,9 @@ module SimpleWorker
|
|
45
47
|
|
46
48
|
def process_ex(ex)
|
47
49
|
body = ex.http_body
|
48
|
-
|
49
|
-
decoded_ex = JSON.parse(
|
50
|
-
exception = Exception.new(ex.message+":"+decoded_ex["msg"])
|
50
|
+
@@logger.debug 'EX BODY=' + body.to_s
|
51
|
+
decoded_ex = JSON.parse(body)
|
52
|
+
exception = Exception.new(ex.message + ": " + decoded_ex["msg"])
|
51
53
|
exception.set_backtrace(decoded_ex["backtrace"].split(",")) if decoded_ex["backtrace"]
|
52
54
|
raise exception
|
53
55
|
end
|
@@ -56,7 +58,7 @@ module SimpleWorker
|
|
56
58
|
begin
|
57
59
|
# ClientHelper.run_http(host, access_key, secret_key, :get, method, nil, params)
|
58
60
|
parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
|
59
|
-
rescue RestClient::
|
61
|
+
rescue RestClient::Exception => ex
|
60
62
|
process_ex(ex)
|
61
63
|
end
|
62
64
|
end
|
@@ -64,7 +66,7 @@ module SimpleWorker
|
|
64
66
|
def post_file(method, file, params={}, options={})
|
65
67
|
begin
|
66
68
|
parse_response RestClient.post(url(method), add_params(method, params).merge!({:file=>file}), :multipart => true), options
|
67
|
-
rescue RestClient::
|
69
|
+
rescue RestClient::Exception => ex
|
68
70
|
process_ex(ex)
|
69
71
|
end
|
70
72
|
end
|
@@ -73,7 +75,7 @@ module SimpleWorker
|
|
73
75
|
begin
|
74
76
|
parse_response RestClient.post(url(method), add_params(method, params).to_json, headers), options
|
75
77
|
#ClientHelper.run_http(host, access_key, secret_key, :post, method, nil, params)
|
76
|
-
rescue RestClient::
|
78
|
+
rescue RestClient::Exception => ex
|
77
79
|
process_ex(ex)
|
78
80
|
end
|
79
81
|
end
|
@@ -83,7 +85,7 @@ module SimpleWorker
|
|
83
85
|
begin
|
84
86
|
parse_response RestClient.put(url(method), add_params(method, body).to_json, headers), options
|
85
87
|
#ClientHelper.run_http(host, access_key, secret_key, :put, method, body, nil)
|
86
|
-
rescue RestClient::
|
88
|
+
rescue RestClient::Exception => ex
|
87
89
|
process_ex(ex)
|
88
90
|
end
|
89
91
|
end
|
@@ -91,7 +93,7 @@ module SimpleWorker
|
|
91
93
|
def delete(method, params={}, options={})
|
92
94
|
begin
|
93
95
|
parse_response RestClient.delete(append_params(url(method), add_params(method, params))), options
|
94
|
-
rescue RestClient::
|
96
|
+
rescue RestClient::Exception => ex
|
95
97
|
process_ex(ex)
|
96
98
|
end
|
97
99
|
end
|
data/lib/simple_worker/base.rb
CHANGED
@@ -238,21 +238,9 @@ module SimpleWorker
|
|
238
238
|
# todo: add a :timeout option
|
239
239
|
def wait_until_complete
|
240
240
|
check_service
|
241
|
-
|
242
|
-
status = nil
|
243
|
-
sleep 1
|
244
|
-
while tries < 100
|
245
|
-
status = self.status
|
246
|
-
puts "Waiting... status=" + status["status"]
|
247
|
-
if status["status"] != "queued" && status["status"] != "running"
|
248
|
-
break
|
249
|
-
end
|
250
|
-
sleep 2
|
251
|
-
end
|
252
|
-
status
|
241
|
+
SimplerWorker.service.wait_until_complete(self.task_id)
|
253
242
|
end
|
254
243
|
|
255
|
-
|
256
244
|
def upload
|
257
245
|
upload_if_needed
|
258
246
|
end
|
File without changes
|
File without changes
|
@@ -1,12 +1,9 @@
|
|
1
|
-
|
2
1
|
# This is used when a bad worker is uploaded.
|
3
|
-
class InvalidWorkerError < StandardError
|
4
|
-
|
5
|
-
end
|
6
|
-
|
7
2
|
|
8
3
|
module SimpleWorker
|
9
4
|
|
5
|
+
class InvalidWorkerError < StandardError; end
|
6
|
+
|
10
7
|
class << self
|
11
8
|
def running_class=(rc)
|
12
9
|
@@running_class = rc
|
@@ -110,7 +107,7 @@ module SimpleWorker
|
|
110
107
|
|
111
108
|
def user_dir
|
112
109
|
# puts 'user_dir=' + @context.user_dir.to_s
|
113
|
-
@
|
110
|
+
@user_dir || "./"
|
114
111
|
end
|
115
112
|
|
116
113
|
def sw_set_data(data)
|
@@ -13,21 +13,35 @@ def init_database_connection(sw_config)
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def get_class_to_run(class_name)
|
16
|
-
runner_class =
|
16
|
+
runner_class = constantize(class_name)
|
17
17
|
return runner_class
|
18
18
|
end
|
19
19
|
|
20
|
-
|
20
|
+
# File activesupport/lib/active_support/inflector/methods.rb, line 107
|
21
|
+
# Shoutout to the MIT License
|
22
|
+
def constantize(camel_cased_word)
|
23
|
+
names = camel_cased_word.split('::')
|
24
|
+
names.shift if names.empty? || names.first.empty?
|
25
|
+
|
26
|
+
constant = Object
|
27
|
+
names.each do |name|
|
28
|
+
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
29
|
+
end
|
30
|
+
constant
|
31
|
+
end
|
32
|
+
|
33
|
+
def init_runner(runner_class, job_data, user_dir)
|
21
34
|
# ensure initialize takes no arguments
|
22
35
|
init_arity = runner_class.instance_method(:initialize).arity
|
23
36
|
if init_arity == 0 || init_arity == -1
|
24
37
|
# good. -1 can be if it's not defined at all
|
25
38
|
else
|
26
|
-
raise InvalidWorkerError, "Worker initialize method must accept zero arguments."
|
39
|
+
raise SimpleWorker::InvalidWorkerError, "Worker initialize method must accept zero arguments."
|
27
40
|
end
|
28
41
|
runner = runner_class.new
|
29
42
|
runner.instance_variable_set(:@job_data, job_data)
|
30
43
|
runner.instance_variable_set(:@sw_config, job_data['sw_config'])
|
44
|
+
runner.instance_variable_set(:@user_dir, user_dir)
|
31
45
|
runner.sw_set_data(job_data)
|
32
46
|
runner
|
33
47
|
end
|
@@ -84,7 +98,7 @@ begin
|
|
84
98
|
SimpleWorker.disable_queueing()
|
85
99
|
runner_class = get_class_to_run(job_data['class_name'])
|
86
100
|
SimpleWorker.running_class = runner_class
|
87
|
-
runner = init_runner(runner_class, job_data)
|
101
|
+
runner = init_runner(runner_class, job_data, dirname)
|
88
102
|
init_worker_service_for_runner(job_data)
|
89
103
|
SimpleWorker.enable_queueing()
|
90
104
|
|
@@ -56,20 +56,9 @@ module SimpleWorker
|
|
56
56
|
begin
|
57
57
|
|
58
58
|
zip_filename = build_merged_file(filename, options[:merge], options[:unmerge], options[:merged_gems], options[:merged_mailers], options[:merged_folders])
|
59
|
-
SimpleWorker.logger.info 'file size to upload: ' + File.size(zip_filename).to_s
|
60
59
|
|
61
60
|
if new_code
|
62
|
-
|
63
|
-
"class_name"=>class_name,
|
64
|
-
"name"=>name,
|
65
|
-
"standalone"=>true,
|
66
|
-
"file_name"=> "runner.rb" # File.basename(filename)
|
67
|
-
}
|
68
|
-
#puts 'options for upload=' + options.inspect
|
69
|
-
SimpleWorker.logger.info "Uploading now..."
|
70
|
-
ret = post_file("code/put", File.new(zip_filename), options)
|
71
|
-
SimpleWorker.logger.info "Done uploading."
|
72
|
-
return ret
|
61
|
+
upload_code(name, zip_filename, 'runner.rb', :runtime=>'ruby')
|
73
62
|
end
|
74
63
|
|
75
64
|
rescue => ex
|
@@ -252,7 +241,7 @@ end
|
|
252
241
|
# todo: should only continue if the gem was auto merged.
|
253
242
|
SimpleWorker.logger.warn "Gem #{gem[:name]} #{gem[:version]} was not found, continuing anyways."
|
254
243
|
else
|
255
|
-
raise "Gem #{gem[:name]} #{gem[:version]} was not found
|
244
|
+
raise "Gem #{gem[:name]} #{gem[:version]} was not found. This will occour when gem_name.gemspec is not the same as the gems primary require."
|
256
245
|
end
|
257
246
|
|
258
247
|
end
|
@@ -292,6 +281,52 @@ end
|
|
292
281
|
fname2
|
293
282
|
end
|
294
283
|
|
284
|
+
# This will package up files into a zip file ready for uploading.
|
285
|
+
def package_code(files)
|
286
|
+
fname2 = "package.zip"
|
287
|
+
File.delete(fname2) if File.exist?(fname2)
|
288
|
+
Zip::ZipFile.open(fname2, 'w') do |f|
|
289
|
+
files.each do |file|
|
290
|
+
f.add(file, file)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
fname2
|
294
|
+
end
|
295
|
+
|
296
|
+
# options:
|
297
|
+
# :runtime => 'ruby', 'python', 'node', 'java', 'go'
|
298
|
+
def upload_code(name, package_file, exec_file, options={})
|
299
|
+
SimpleWorker.logger.info 'file size to upload: ' + File.size(package_file).to_s
|
300
|
+
options = {
|
301
|
+
"name"=>name,
|
302
|
+
"class_name"=>name, # todo: remove this shortly
|
303
|
+
"standalone"=>true,
|
304
|
+
"runtime"=>options[:runtime],
|
305
|
+
"file_name"=> exec_file # File.basename(filename)
|
306
|
+
}
|
307
|
+
#puts 'options for upload=' + options.inspect
|
308
|
+
SimpleWorker.logger.info "Uploading now..."
|
309
|
+
ret = post_file("code/put", File.new(package_file), options)
|
310
|
+
SimpleWorker.logger.info "Done uploading."
|
311
|
+
return ret
|
312
|
+
|
313
|
+
end
|
314
|
+
|
315
|
+
def wait_until_complete(task_id)
|
316
|
+
tries = 0
|
317
|
+
status = nil
|
318
|
+
sleep 1
|
319
|
+
while tries < 100
|
320
|
+
status = status(task_id)
|
321
|
+
puts "Waiting... status=" + status["status"]
|
322
|
+
if status["status"] != "queued" && status["status"] != "running"
|
323
|
+
break
|
324
|
+
end
|
325
|
+
sleep 2
|
326
|
+
end
|
327
|
+
status
|
328
|
+
end
|
329
|
+
|
295
330
|
def add_sw_params(hash_to_send)
|
296
331
|
# todo: remove secret key?? Can use worker service from within a worker without it now
|
297
332
|
hash_to_send["sw_access_key"] = self.access_key
|
data/lib/simple_worker.rb
CHANGED
@@ -40,8 +40,8 @@ if defined?(Rails)
|
|
40
40
|
# puts 'Rails=' + Rails.inspect
|
41
41
|
# puts 'vers=' + Rails::VERSION::MAJOR.inspect
|
42
42
|
if Rails::VERSION::MAJOR == 2
|
43
|
-
require_relative 'rails2_init.rb'
|
43
|
+
require_relative 'simple_worker/rails2_init.rb'
|
44
44
|
else
|
45
|
-
require_relative 'railtie'
|
45
|
+
require_relative 'simple_worker/railtie'
|
46
46
|
end
|
47
47
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: simple_worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 1.0.
|
5
|
+
version: 1.0.6
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Travis Reeder
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-08-
|
13
|
+
date: 2011-08-26 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: zip
|
@@ -43,12 +43,13 @@ extensions: []
|
|
43
43
|
extra_rdoc_files:
|
44
44
|
- README.markdown
|
45
45
|
files:
|
46
|
-
- lib/
|
47
|
-
- lib/railtie.rb
|
46
|
+
- lib/generators/simple_worker/simple_worker_generator.rb
|
48
47
|
- lib/simple_worker.rb
|
49
48
|
- lib/simple_worker/api.rb
|
50
49
|
- lib/simple_worker/base.rb
|
51
50
|
- lib/simple_worker/config.rb
|
51
|
+
- lib/simple_worker/rails2_init.rb
|
52
|
+
- lib/simple_worker/railtie.rb
|
52
53
|
- lib/simple_worker/server/overrides.rb
|
53
54
|
- lib/simple_worker/server/runner.rb
|
54
55
|
- lib/simple_worker/service.rb
|
@@ -79,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
79
80
|
requirements: []
|
80
81
|
|
81
82
|
rubyforge_project:
|
82
|
-
rubygems_version: 1.8.
|
83
|
+
rubygems_version: 1.8.8
|
83
84
|
signing_key:
|
84
85
|
specification_version: 3
|
85
86
|
summary: The official SimpleWorker gem for http://www.simpleworker.com
|