iron_worker 2.4.3 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +352 -0
- data/iron_worker.gemspec +3 -8
- data/lib/iron_worker.rb +13 -51
- data/lib/iron_worker/api_client.rb +152 -0
- data/lib/iron_worker/version.rb +9 -1
- data/lib/iron_worker/worker_helper.rb +72 -0
- metadata +18 -183
- data/README.markdown +0 -73
- data/lib/generators/iron_worker/iron_worker_generator.rb +0 -13
- data/lib/generators/iron_worker/templates/template_worker.erb +0 -7
- data/lib/iron_worker/api.rb +0 -244
- data/lib/iron_worker/base.rb +0 -449
- data/lib/iron_worker/config.rb +0 -287
- data/lib/iron_worker/rails2_init.rb +0 -8
- data/lib/iron_worker/railtie.rb +0 -16
- data/lib/iron_worker/server/overrides.rb +0 -198
- data/lib/iron_worker/server/runner.rb +0 -82
- data/lib/iron_worker/service.rb +0 -644
- data/lib/iron_worker/used_in_worker.rb +0 -11
- data/lib/iron_worker/utils.rb +0 -11
- data/rails/init.rb +0 -0
- data/test/Gemfile +0 -36
- data/test/Gemfile.lock +0 -169
- data/test/README.md +0 -5
- data/test/batch_run.rb +0 -121
- data/test/config_sample.yml +0 -13
- data/test/models/cool_model.rb +0 -7
- data/test/models/db_model.rb +0 -6
- data/test/models/model_1.rb +0 -16
- data/test/models/model_2.rb +0 -16
- data/test/quick_run.rb +0 -36
- data/test/quick_schedule.rb +0 -18
- data/test/resources/something.yml +0 -4
- data/test/simple_script.rb +0 -11
- data/test/test_base.rb +0 -79
- data/test/test_database.rb +0 -31
- data/test/test_gems.rb +0 -31
- data/test/test_inheritance.rb +0 -25
- data/test/test_iron_worker.rb +0 -194
- data/test/test_merging.rb +0 -51
- data/test/test_non_worker_script.rb +0 -14
- data/test/test_rails.rb +0 -35
- data/test/test_scheduling.rb +0 -74
- data/test/test_scheduling_no_active_support.rb +0 -37
- data/test/test_service.rb +0 -28
- data/test/test_uploads.rb +0 -30
- data/test/test_webhooks.rb +0 -45
- data/test/v2quick.rb +0 -29
- data/test/workers/awesome_job.rb +0 -26
- data/test/workers/aws_s3_worker.rb +0 -10
- data/test/workers/big_data_worker.rb +0 -15
- data/test/workers/big_gems_worker.rb +0 -35
- data/test/workers/cool_worker.rb +0 -14
- data/test/workers/db_worker.rb +0 -27
- data/test/workers/fail_worker.rb +0 -10
- data/test/workers/gem_dependency_worker.rb +0 -20
- data/test/workers/hacker_worker.rb +0 -65
- data/test/workers/iw_test_worker.rb +0 -22
- data/test/workers/iw_test_worker_2.rb +0 -53
- data/test/workers/iw_test_worker_3.rb +0 -17
- data/test/workers/local_vs_remote_worker.rb +0 -9
- data/test/workers/merging_worker.rb +0 -40
- data/test/workers/mq_worker.rb +0 -21
- data/test/workers/one_line_worker.rb +0 -10
- data/test/workers/prawn_worker.rb +0 -32
- data/test/workers/progress_worker.rb +0 -26
- data/test/workers/qb_worker.rb +0 -17
- data/test/workers/rails_worker.rb +0 -24
- data/test/workers/rails_worker2.rb +0 -12
- data/test/workers/requiring_worker.rb +0 -4
- data/test/workers/running_back_worker.rb +0 -7
- data/test/workers/scheduled_worker.rb +0 -15
- data/test/workers/second_worker.rb +0 -16
- data/test/workers/third_worker.rb +0 -6
- data/test/workers/webhook_worker.rb +0 -7
- data/test/zip_vs_gzip.rb +0 -26
data/README.markdown
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
Getting Started
|
2
|
-
===============
|
3
|
-
|
4
|
-
[Sign up for a IronWorker account][1], it's free to try!
|
5
|
-
|
6
|
-
[1]: http://www.iron.io/
|
7
|
-
|
8
|
-
**WARNING**: THIS GEM IS DEPRECATED. Use the generation IronWorker gem, `iron_worker_ng`,
|
9
|
-
at: https://github.com/iron-io/iron_worker_ruby_ng. It also includes our new IronWorker command line
|
10
|
-
interface (CLI) which is... awesome. ;) You can [read more about that here](http://blog.iron.io/2012/05/new-ironworker-command-line-interface.html).
|
11
|
-
|
12
|
-
Install IronWorker Gem
|
13
|
-
------------------------
|
14
|
-
|
15
|
-
gem install iron_worker
|
16
|
-
|
17
|
-
Configure IronWorker
|
18
|
-
----------------------
|
19
|
-
|
20
|
-
You really just need your token, which you can get [here][2]
|
21
|
-
[2]: http://hud.iron.io/tokens
|
22
|
-
|
23
|
-
IronWorker.configure do |config|
|
24
|
-
config.token = TOKEN
|
25
|
-
config.project_id = MY_PROJECT_ID
|
26
|
-
end
|
27
|
-
|
28
|
-
Write a Worker
|
29
|
-
--------------
|
30
|
-
|
31
|
-
Here's an example worker that sends an email:
|
32
|
-
|
33
|
-
require 'iron_worker'
|
34
|
-
|
35
|
-
class HelloWorker < IronWorker::Base
|
36
|
-
|
37
|
-
attr_accessor :name
|
38
|
-
|
39
|
-
# This is the method that will be run
|
40
|
-
def run
|
41
|
-
puts "Hello #{name}!"
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
Test It Locally
|
46
|
-
---------------
|
47
|
-
|
48
|
-
Let's say someone does something in your app and you want to send an email about it.
|
49
|
-
|
50
|
-
worker = HelloWorker.new
|
51
|
-
worker.name = "Travis"
|
52
|
-
worker.run_local
|
53
|
-
|
54
|
-
Once you've got it working locally, the next step is to run it on the IronWorker cloud.
|
55
|
-
|
56
|
-
Queue up your Worker on the IronWorker Cloud
|
57
|
-
----------------------------------------------
|
58
|
-
|
59
|
-
Let's say someone does something in your app and you want to send an email about it.
|
60
|
-
|
61
|
-
worker = HelloWorker.new
|
62
|
-
worker.name = "Travis"
|
63
|
-
worker.queue
|
64
|
-
|
65
|
-
This will send it off to the IronWorker cloud.
|
66
|
-
|
67
|
-
Full Documentation
|
68
|
-
-----------------
|
69
|
-
|
70
|
-
Now that you've got your first worker running, be sure to [check out the full documentation](https://github.com/iron-io/iron_worker_ruby/wiki).
|
71
|
-
IronWorker can do so much more!
|
72
|
-
|
73
|
-
And check out the [Iron.io Dev Center](http://dev.iron.io) for full IronWorker API documentation and and support for other languages.
|
@@ -1,13 +0,0 @@
|
|
1
|
-
class IronWorkerGenerator < Rails::Generators::NamedBase
|
2
|
-
source_root File.expand_path("../templates", __FILE__)
|
3
|
-
|
4
|
-
desc "Creates a new skeleton worker - NAME is camelized"
|
5
|
-
def create_worker_file
|
6
|
-
# file_name needs to be classified
|
7
|
-
@camel = file_name.camelize
|
8
|
-
if not File.directory? "#{Rails.root}/app/workers"
|
9
|
-
Dir.mkdir "#{Rails.root}/app/workers"
|
10
|
-
end
|
11
|
-
template "template_worker.erb", "app/workers/#{file_name}.rb"
|
12
|
-
end
|
13
|
-
end
|
data/lib/iron_worker/api.rb
DELETED
@@ -1,244 +0,0 @@
|
|
1
|
-
require 'rest'
|
2
|
-
|
3
|
-
module IronWorker
|
4
|
-
|
5
|
-
class RequestError < StandardError
|
6
|
-
def initialize(msg, options={})
|
7
|
-
super(msg)
|
8
|
-
@options = options
|
9
|
-
end
|
10
|
-
|
11
|
-
def status
|
12
|
-
@options[:status]
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
module Api
|
17
|
-
|
18
|
-
module Signatures
|
19
|
-
|
20
|
-
|
21
|
-
def self.generate_timestamp(gmtime)
|
22
|
-
return gmtime.strftime("%Y-%m-%dT%H:%M:%SZ")
|
23
|
-
end
|
24
|
-
|
25
|
-
|
26
|
-
def self.generate_signature(operation, timestamp, secret_key)
|
27
|
-
my_sha_hmac = Digest::HMAC.digest(operation + timestamp, secret_key, Digest::SHA1)
|
28
|
-
my_b64_hmac_digest = Base64.encode64(my_sha_hmac).strip
|
29
|
-
return my_b64_hmac_digest
|
30
|
-
end
|
31
|
-
|
32
|
-
|
33
|
-
def self.hash_to_s(hash)
|
34
|
-
str = ""
|
35
|
-
hash.sort.each { |a| str+= "#{a[0]}#{a[1]}" }
|
36
|
-
#removing all characters that could differ after parsing with rails
|
37
|
-
return str.delete "\"\/:{}[]\' T"
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
# Subclass must define:
|
42
|
-
# host: endpoint url for service
|
43
|
-
class Client
|
44
|
-
|
45
|
-
attr_accessor :scheme, :host, :port, :token, :version, :config
|
46
|
-
|
47
|
-
def initialize(host, token, options={})
|
48
|
-
@config = options[:config]
|
49
|
-
@scheme = options[:scheme] || @config.scheme || "https"
|
50
|
-
@host = options[:host] || @config.host || host
|
51
|
-
@port = options[:port] || @config.port || 443
|
52
|
-
@token = options[:token] || @config.token || token
|
53
|
-
@version = options[:version]
|
54
|
-
#@logger = options[:logger]
|
55
|
-
|
56
|
-
reset_base_url
|
57
|
-
|
58
|
-
@uber_client = Rest::Client.new
|
59
|
-
|
60
|
-
end
|
61
|
-
|
62
|
-
def reset_base_url
|
63
|
-
@base_url = "#{@scheme}://#{@host}:#{@port}/#{@version}"
|
64
|
-
end
|
65
|
-
|
66
|
-
def base_url
|
67
|
-
@base_url
|
68
|
-
end
|
69
|
-
|
70
|
-
def url(command_path)
|
71
|
-
# @logger.debug "url: " + url.to_s
|
72
|
-
"/#{command_path}"
|
73
|
-
end
|
74
|
-
|
75
|
-
def url_full(command_path)
|
76
|
-
url = "#{base_url}/#{command_path}"
|
77
|
-
# @logger.debug "url: " + url.to_s
|
78
|
-
url
|
79
|
-
end
|
80
|
-
|
81
|
-
|
82
|
-
def common_req_hash
|
83
|
-
{
|
84
|
-
:headers=>{"Content-Type" => 'application/json',
|
85
|
-
"Authorization"=>"OAuth #{@token}",
|
86
|
-
"User-Agent"=>"IronWorker Ruby Client"}
|
87
|
-
}
|
88
|
-
end
|
89
|
-
|
90
|
-
def process_ex(ex)
|
91
|
-
logger.error "EX #{ex.class.name}: #{ex.message}"
|
92
|
-
body = ex.http_body
|
93
|
-
logger.debug 'EX http_code: ' + ex.http_code.to_s
|
94
|
-
logger.debug 'EX BODY=' + body.to_s
|
95
|
-
decoded_ex = JSON.parse(body)
|
96
|
-
exception = Exception.new(ex.message + ": " + decoded_ex["msg"])
|
97
|
-
exception.set_backtrace(decoded_ex["backtrace"].split(",")) if decoded_ex["backtrace"]
|
98
|
-
raise exception
|
99
|
-
end
|
100
|
-
|
101
|
-
|
102
|
-
def check_response(response, options={})
|
103
|
-
# response.code # http status code
|
104
|
-
#response.time # time in seconds the request took
|
105
|
-
#response.headers # the http headers
|
106
|
-
#response.headers_hash # http headers put into a hash
|
107
|
-
#response.body # the response body
|
108
|
-
status = response.code
|
109
|
-
body = response.body
|
110
|
-
# todo: check content-type == application/json before parsing
|
111
|
-
logger.debug "response code=" + status.to_s
|
112
|
-
logger.debug "response body=" + body.inspect
|
113
|
-
res = nil
|
114
|
-
unless options[:parse] == false
|
115
|
-
res = JSON.parse(body)
|
116
|
-
end
|
117
|
-
if status < 400
|
118
|
-
|
119
|
-
else
|
120
|
-
raise IronWorker::RequestError.new((res ? "#{status}: #{res["msg"]}" : "#{status} Error! parse=false so no msg"), :status=>status)
|
121
|
-
end
|
122
|
-
res || body
|
123
|
-
end
|
124
|
-
|
125
|
-
def get(method, params={}, options={})
|
126
|
-
full_url = url_full(method)
|
127
|
-
#all_params = add_params(method, params)
|
128
|
-
#url_plus_params = append_params(full_url, all_params)
|
129
|
-
logger.debug 'get url=' + full_url
|
130
|
-
req_hash = common_req_hash
|
131
|
-
req_hash[:params] = params
|
132
|
-
response = @uber_client.get(full_url, req_hash) # could let typhoeus add params, using :params=>x
|
133
|
-
#response = @http_sess.request(:get, url_plus_params,
|
134
|
-
# {},
|
135
|
-
# {})
|
136
|
-
check_response(response, options)
|
137
|
-
body = response.body
|
138
|
-
parse_response(body, options)
|
139
|
-
|
140
|
-
end
|
141
|
-
|
142
|
-
def post_file(method, file, params={}, options={})
|
143
|
-
begin
|
144
|
-
data = add_params(method, params).to_json
|
145
|
-
url = url_full(method)
|
146
|
-
logger.debug "post_file url = " + url
|
147
|
-
logger.debug "data = " + data
|
148
|
-
logger.debug "params = " + params.inspect
|
149
|
-
logger.debug "options = " + options.inspect
|
150
|
-
# todo: replace with uber_client
|
151
|
-
parse_response(RestClient.post(append_params(url, add_params(method, params)), {:data => data, :file => file}, :content_type => 'application/json'), options)
|
152
|
-
rescue RestClient::Exception => ex
|
153
|
-
process_ex(ex)
|
154
|
-
end
|
155
|
-
end
|
156
|
-
|
157
|
-
def post(method, params={}, options={})
|
158
|
-
logger.debug "params = " + params.inspect
|
159
|
-
logger.debug "options = " + options.inspect
|
160
|
-
logger.debug "params.payload = " + params[:payload].inspect
|
161
|
-
logger.debug "token = "+ token.inspect
|
162
|
-
begin
|
163
|
-
url = url_full(method)
|
164
|
-
logger.debug 'post url=' + url
|
165
|
-
json = add_params(method, params).to_json
|
166
|
-
logger.debug 'body=' + json
|
167
|
-
req_hash = common_req_hash
|
168
|
-
req_hash[:body] = json
|
169
|
-
response = @uber_client.post(url, req_hash)
|
170
|
-
#response = @http_sess.post(url, json, {"Content-Type" => 'application/json'})
|
171
|
-
check_response(response)
|
172
|
-
logger.debug 'response: ' + response.inspect
|
173
|
-
body = response.body
|
174
|
-
parse_response(body, options)
|
175
|
-
rescue IronWorker::RequestError => ex
|
176
|
-
# let it throw, came from check_response
|
177
|
-
raise ex
|
178
|
-
rescue RestClient::Exception => ex
|
179
|
-
logger.warn("Exception in post! #{ex.message}")
|
180
|
-
logger.warn(ex.backtrace.join("\n"))
|
181
|
-
process_ex(ex)
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
|
186
|
-
def put(method, body, options={})
|
187
|
-
begin
|
188
|
-
# todo: replace with uber_client
|
189
|
-
parse_response RestClient.put(url_full(method), add_params(method, body).to_json, headers), options
|
190
|
-
rescue RestClient::Exception => ex
|
191
|
-
process_ex(ex)
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
def delete(method, params={}, options={})
|
196
|
-
begin
|
197
|
-
# todo: replace with uber_client
|
198
|
-
parse_response RestClient.delete(append_params(url_full(method), add_params(method, params))), options
|
199
|
-
rescue RestClient::Exception => ex
|
200
|
-
process_ex(ex)
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
def add_params(command_path, hash)
|
205
|
-
extra_params = {'oauth' => token}
|
206
|
-
hash.merge!(extra_params)
|
207
|
-
end
|
208
|
-
|
209
|
-
def append_params(host, params)
|
210
|
-
host += "?"
|
211
|
-
i = 0
|
212
|
-
params.each_pair do |k, v|
|
213
|
-
#puts "k=#{k} v=#{v}"
|
214
|
-
host += "&" if i > 0
|
215
|
-
host += k + "=" + (v.is_a?(String) ? CGI.escape(v) : v.to_s)
|
216
|
-
i +=1
|
217
|
-
end
|
218
|
-
return host
|
219
|
-
end
|
220
|
-
|
221
|
-
def headers
|
222
|
-
user_agent = "IronWorker Ruby Client"
|
223
|
-
headers = {'User-Agent' => user_agent}
|
224
|
-
end
|
225
|
-
|
226
|
-
def parse_response(response, options={})
|
227
|
-
#puts 'PARSE RESPONSE: ' + response.to_s
|
228
|
-
unless options[:parse] == false
|
229
|
-
begin
|
230
|
-
return JSON.parse(response.to_s)
|
231
|
-
rescue => ex
|
232
|
-
puts 'parse_response: response that caused error = ' + response.to_s
|
233
|
-
raise ex
|
234
|
-
end
|
235
|
-
else
|
236
|
-
response
|
237
|
-
end
|
238
|
-
end
|
239
|
-
|
240
|
-
end
|
241
|
-
|
242
|
-
end
|
243
|
-
|
244
|
-
end
|
data/lib/iron_worker/base.rb
DELETED
@@ -1,449 +0,0 @@
|
|
1
|
-
require 'digest/md5'
|
2
|
-
require 'base64'
|
3
|
-
|
4
|
-
module IronWorker
|
5
|
-
|
6
|
-
class Base
|
7
|
-
|
8
|
-
attr_accessor :task_set_id, :task_id, :schedule_id
|
9
|
-
attr_reader :response
|
10
|
-
|
11
|
-
class << self
|
12
|
-
attr_accessor :subclass, :caller_file
|
13
|
-
@merged = {}
|
14
|
-
@merged_workers = {}
|
15
|
-
@merged_gems = {}
|
16
|
-
@merged_mailers = {}
|
17
|
-
@merged_folders = {}
|
18
|
-
@unmerged = {}
|
19
|
-
@unmerged_gems = {}
|
20
|
-
|
21
|
-
def reset!
|
22
|
-
@merged = {}
|
23
|
-
@merged_workers = {}
|
24
|
-
@merged_gems = {}
|
25
|
-
@merged_mailers = {}
|
26
|
-
@merged_folders = {}
|
27
|
-
@unmerged = {}
|
28
|
-
@unmerged_gems = {}
|
29
|
-
end
|
30
|
-
|
31
|
-
def inherited(subclass)
|
32
|
-
subclass.reset!
|
33
|
-
|
34
|
-
caller_file = caller[0][0...(caller[0].index(":in"))]
|
35
|
-
caller_file = caller_file[0...(caller_file.rindex(":"))]
|
36
|
-
subclass.instance_variable_set(:@caller_file, caller_file)
|
37
|
-
|
38
|
-
super
|
39
|
-
end
|
40
|
-
|
41
|
-
# merges the specified gem.
|
42
|
-
def merge_gem(gem_name, options={})
|
43
|
-
gem_info = IronWorker::MergeHelper.create_gem_info(gem_name, options)
|
44
|
-
@merged_gems[gem_name.to_s] = gem_info
|
45
|
-
reqs = gem_info[:require].is_a?(Array) ? gem_info[:require] : [gem_info[:require]]
|
46
|
-
reqs.each do |r|
|
47
|
-
r2 = "#{gem_info[:path]}/lib/#{r}"
|
48
|
-
begin
|
49
|
-
IronWorker.logger.debug 'requiring ' + r2
|
50
|
-
require r2
|
51
|
-
rescue LoadError=>ex
|
52
|
-
IronWorker.logger.error "Error requiring gem #{r}: #{ex.message}"
|
53
|
-
raise "Gem #{gem_name} was found, but we could not load the file '#{r2}'. You may need to use :require=>x.........."
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
|
59
|
-
def unmerge_gem(gem_name)
|
60
|
-
#gem_info = {:name=>gem_name}
|
61
|
-
#@unmerged_gems[gem_name.to_s] = gem_info
|
62
|
-
gs = gem_name.to_s
|
63
|
-
gem_info = {:name=>gs}
|
64
|
-
@unmerged_gems[gs] = gem_info
|
65
|
-
@merged_gems.delete(gs)
|
66
|
-
end
|
67
|
-
|
68
|
-
#merge action_mailer mailers
|
69
|
-
def merge_mailer(mailer, params={})
|
70
|
-
f2 = IronWorker::MergeHelper.check_for_file(mailer, @caller_file)
|
71
|
-
basename = File.basename(mailer, f2[:extname])
|
72
|
-
path_to_templates = params[:path_to_templates] || File.join(Rails.root, "app/views/#{basename}")
|
73
|
-
path_to_layouts = params[:path_to_layouts] || File.join(Rails.root, "app/views/layouts")
|
74
|
-
@merged_mailers[basename] = {:name => basename, :path_to_templates => path_to_templates, :path_to_layouts => path_to_layouts, :filename => f2[:path]}.merge!(params)
|
75
|
-
end
|
76
|
-
|
77
|
-
|
78
|
-
def merge_folder(path)
|
79
|
-
files = []
|
80
|
-
#puts "caller_file=" + caller_file
|
81
|
-
if path[0, 1] == '/'
|
82
|
-
abs_dir = path
|
83
|
-
else # relative
|
84
|
-
abs_dir = File.join(File.dirname(caller_file), path)
|
85
|
-
end
|
86
|
-
#puts 'abs_dir=' + abs_dir
|
87
|
-
raise "Folder not found for merge_folder #{path}!" unless File.directory?(abs_dir)
|
88
|
-
rbfiles = File.join(abs_dir, "*.rb")
|
89
|
-
Dir[rbfiles].each do |f|
|
90
|
-
#f2 = check_for_file(f)
|
91
|
-
#puts "f2=#{f2}"
|
92
|
-
merge(f)
|
93
|
-
#files << f
|
94
|
-
#@merged[f]
|
95
|
-
end
|
96
|
-
#@merged_folders[path] = files unless files.empty?
|
97
|
-
#IronWorker.logger.info "Merged folders! #{@merged_folders.inspect}"
|
98
|
-
end
|
99
|
-
|
100
|
-
# merges the specified file.
|
101
|
-
#
|
102
|
-
# Example: merge 'models/my_model'
|
103
|
-
def merge(f)
|
104
|
-
f2 = IronWorker::MergeHelper.check_for_file(f, @caller_file)
|
105
|
-
fbase = f2[:basename]
|
106
|
-
ret = f2
|
107
|
-
@merged[fbase] = ret
|
108
|
-
ret
|
109
|
-
end
|
110
|
-
|
111
|
-
# Opposite of merge, this will omit the files you specify from being merged in. Useful in Rails apps
|
112
|
-
# where a lot of things are auto-merged by default like your models.
|
113
|
-
def unmerge(f)
|
114
|
-
f2 = IronWorker::MergeHelper.check_for_file(f, @caller_file)
|
115
|
-
fbase = f2[:basename]
|
116
|
-
@unmerged[fbase] = f2
|
117
|
-
@merged.delete(fbase)
|
118
|
-
end
|
119
|
-
|
120
|
-
|
121
|
-
# Use this to merge in other workers. These are treated differently the normal merged files because
|
122
|
-
# they will be uploaded separately and treated as distinctly separate workers.
|
123
|
-
#
|
124
|
-
# @param file [String] This is the path to the file, just like merge.
|
125
|
-
# @param class_name [String|Class] 'MyWorker' or just MyWorker.
|
126
|
-
# @return [Hash]
|
127
|
-
def merge_worker(file, class_name)
|
128
|
-
ret = merge(file)
|
129
|
-
ret[:class_name] = case class_name
|
130
|
-
when String
|
131
|
-
class_name.strip
|
132
|
-
when Class
|
133
|
-
class_name.name
|
134
|
-
else
|
135
|
-
IronWorker.service.logger.warn "merge_worker: only String or Class is expected as class_name"
|
136
|
-
class_name # probably user does know what is he doing
|
137
|
-
end
|
138
|
-
@merged_workers[file] = ret
|
139
|
-
ret
|
140
|
-
end
|
141
|
-
end
|
142
|
-
|
143
|
-
|
144
|
-
def log(str)
|
145
|
-
puts str.to_s
|
146
|
-
end
|
147
|
-
|
148
|
-
def user_dir
|
149
|
-
"./"
|
150
|
-
end
|
151
|
-
|
152
|
-
def set_progress(hash)
|
153
|
-
puts 'set_progress: ' + hash.inspect
|
154
|
-
end
|
155
|
-
|
156
|
-
def who_am_i?
|
157
|
-
return self.class.name
|
158
|
-
end
|
159
|
-
|
160
|
-
def uploaded?
|
161
|
-
self.class.instance_variable_defined?(:@uploaded) && self.class.instance_variable_get(:@uploaded)
|
162
|
-
end
|
163
|
-
|
164
|
-
# Call this if you want to run locally and get some extra features from this gem like global attributes.
|
165
|
-
def run_local
|
166
|
-
# puts 'run_local'
|
167
|
-
set_auto_attributes
|
168
|
-
init_database
|
169
|
-
init_mailer
|
170
|
-
begin
|
171
|
-
run
|
172
|
-
rescue => ex
|
173
|
-
if self.respond_to?(:rescue_all)
|
174
|
-
rescue_all(ex)
|
175
|
-
else
|
176
|
-
raise ex
|
177
|
-
end
|
178
|
-
end
|
179
|
-
end
|
180
|
-
|
181
|
-
def init_mailer
|
182
|
-
if IronWorker.config.mailer
|
183
|
-
require 'action_mailer'
|
184
|
-
ActionMailer::Base.raise_delivery_errors = true
|
185
|
-
ActionMailer::Base.smtp_settings = (IronWorker.config.mailer)
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
def init_database
|
190
|
-
if IronWorker.config.database
|
191
|
-
require 'active_record'
|
192
|
-
if !ActiveRecord::Base.connected?
|
193
|
-
ActiveRecord::Base.establish_connection(IronWorker.config.database)
|
194
|
-
end
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
def set_auto_attributes
|
199
|
-
set_global_attributes
|
200
|
-
end
|
201
|
-
|
202
|
-
def set_global_attributes
|
203
|
-
return unless IronWorker.config
|
204
|
-
ga = IronWorker.config.global_attributes
|
205
|
-
if ga && ga.size > 0
|
206
|
-
ga.each_pair do |k, v|
|
207
|
-
# puts "k=#{k} v=#{v}"
|
208
|
-
if self.respond_to?(k)
|
209
|
-
self.send("#{k}=", v)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
def enqueue(options={})
|
216
|
-
queue(options)
|
217
|
-
end
|
218
|
-
|
219
|
-
# Call this to queue up your job to IronWorker cloud.
|
220
|
-
# options:
|
221
|
-
# :priority => 0, 1 or 2. Default is 0.
|
222
|
-
# :recursive => true/false. Default is false. If you queue up a worker that is the same class as the currently
|
223
|
-
# running worker, it will be rejected unless you set this explicitly so we know you meant to do it.
|
224
|
-
def queue(options={})
|
225
|
-
# puts 'in queue'
|
226
|
-
|
227
|
-
IronWorker.config.force_upload = IronWorker.config.force_upload && is_local?
|
228
|
-
set_auto_attributes
|
229
|
-
upload_if_needed(options)
|
230
|
-
response = IronWorker.service.queue(self.class.name, sw_get_data, options)
|
231
|
-
IronWorker.service.logger.debug 'queue response=' + response.inspect
|
232
|
-
@response = response
|
233
|
-
@task_id = response["tasks"][0]["id"]
|
234
|
-
response
|
235
|
-
end
|
236
|
-
|
237
|
-
# Receive the status of your worker.
|
238
|
-
def status
|
239
|
-
check_service
|
240
|
-
if task_id
|
241
|
-
task_status
|
242
|
-
elsif schedule_id
|
243
|
-
schedule_status
|
244
|
-
else
|
245
|
-
raise "Queue or schedule before check status."
|
246
|
-
end
|
247
|
-
end
|
248
|
-
|
249
|
-
def task_status
|
250
|
-
IronWorker.service.status(task_id)
|
251
|
-
end
|
252
|
-
|
253
|
-
def is_local?
|
254
|
-
IronWorker.is_local?
|
255
|
-
end
|
256
|
-
|
257
|
-
def is_remote?
|
258
|
-
IronWorker.is_remote?
|
259
|
-
end
|
260
|
-
|
261
|
-
# will return after job has completed or errored out.
|
262
|
-
# Returns status.
|
263
|
-
# todo: add a :timeout option
|
264
|
-
def wait_until_complete
|
265
|
-
check_service
|
266
|
-
raise "No ID for worker!" if self.task_id.nil? && self.schedule_id.nil?
|
267
|
-
IronWorker.service.wait_until_complete(self.task_id || self.schedule_id, :schedule=>is_schedule?)
|
268
|
-
end
|
269
|
-
|
270
|
-
def is_schedule?
|
271
|
-
!self.schedule_id.nil?
|
272
|
-
end
|
273
|
-
|
274
|
-
def upload
|
275
|
-
upload_if_needed
|
276
|
-
end
|
277
|
-
|
278
|
-
#
|
279
|
-
# schedule: hash of scheduling options that can include:
|
280
|
-
# Required:
|
281
|
-
# - start_at: Time of first run - DateTime or Time object.
|
282
|
-
# Optional:
|
283
|
-
# - run_every: Time in seconds between runs. If ommitted, task will only run once.
|
284
|
-
# - delay_type: Fixed Rate or Fixed Delay. Default is fixed_delay.
|
285
|
-
# - end_at: Scheduled task will stop running after this date (optional, if ommitted, runs forever or until cancelled)
|
286
|
-
# - run_times: Task will run exactly :run_times. For instance if :run_times is 5, then the task will run 5 times.
|
287
|
-
# - name: Provide a name for the schedule, defaults to class name. Use this if you want more than one schedule per worker class.
|
288
|
-
#
|
289
|
-
def schedule(schedule)
|
290
|
-
set_global_attributes
|
291
|
-
upload_if_needed(schedule)
|
292
|
-
|
293
|
-
response = IronWorker.service.schedule(self.class.name, sw_get_data, schedule)
|
294
|
-
IronWorker.service.logger.debug 'schedule response=' + response.inspect
|
295
|
-
@schedule_id = response["schedules"][0]["id"]
|
296
|
-
response
|
297
|
-
end
|
298
|
-
|
299
|
-
def schedule_status
|
300
|
-
IronWorker.service.schedule_status(schedule_id)
|
301
|
-
end
|
302
|
-
|
303
|
-
# Retrieves the log for this worker from the IronWorker service.
|
304
|
-
def get_log(options={})
|
305
|
-
IronWorker.service.log(task_id, options)
|
306
|
-
end
|
307
|
-
|
308
|
-
# Callbacks for developer
|
309
|
-
def before_upload
|
310
|
-
|
311
|
-
end
|
312
|
-
|
313
|
-
def after_upload
|
314
|
-
|
315
|
-
end
|
316
|
-
|
317
|
-
def before_run
|
318
|
-
|
319
|
-
end
|
320
|
-
|
321
|
-
def after_run
|
322
|
-
|
323
|
-
end
|
324
|
-
|
325
|
-
private
|
326
|
-
|
327
|
-
def gems_to_merge(merged_gems)
|
328
|
-
list_of_gems = {}
|
329
|
-
if merged_gems && merged_gems.size > 0
|
330
|
-
installed_gems = IronWorker.config.get_server_gems
|
331
|
-
merged_gems.each_pair do |k, gem|
|
332
|
-
gem.merge!({:merge=>(!installed_gems.find { |g| g["name"]==gem[:name] && g["version"]==gem[:version] })})
|
333
|
-
list_of_gems[gem[:name]] = gem # don't' need this if (list_of_gems.select { |k,g| g[:name]==gem[:name] }).empty?
|
334
|
-
end
|
335
|
-
IronWorker.logger.debug "#{list_of_gems.inspect}"
|
336
|
-
end
|
337
|
-
list_of_gems
|
338
|
-
end
|
339
|
-
|
340
|
-
def check_service
|
341
|
-
raise "IronWorker configuration not set." unless IronWorker.service
|
342
|
-
end
|
343
|
-
|
344
|
-
def self.extract_superclasses_merges(worker, merged)
|
345
|
-
subclass = worker.class
|
346
|
-
rfile = subclass.instance_variable_get(:@caller_file) # Base.caller_file # File.expand_path(Base.subclass)
|
347
|
-
# puts 'subclass file=' + rfile.inspect
|
348
|
-
# puts 'subclass.name=' + subclass.name
|
349
|
-
superclass = subclass
|
350
|
-
# Also get merged from subclasses up to IronWorker::Base
|
351
|
-
while (superclass = superclass.superclass)
|
352
|
-
#puts 'superclass=' + superclass.name
|
353
|
-
break if superclass.name == IronWorker::Base.name
|
354
|
-
super_merged = superclass.instance_variable_get(:@merged)
|
355
|
-
#puts 'merging caller file: ' + superclass.instance_variable_get(:@caller_file).inspect
|
356
|
-
caller_to_add = superclass.instance_variable_get(:@caller_file)
|
357
|
-
fb = File.basename(caller_to_add)
|
358
|
-
r = {:name=>fb, :path=>caller_to_add}
|
359
|
-
super_merged[fb] = r
|
360
|
-
merged.merge!(super_merged)
|
361
|
-
#puts 'merged with superclass=' + merged.inspect
|
362
|
-
end
|
363
|
-
return merged, rfile, subclass
|
364
|
-
end
|
365
|
-
|
366
|
-
def self.extract_merged_workers(worker)
|
367
|
-
merged_workers = worker.class.instance_variable_get(:@merged_workers)
|
368
|
-
IronWorker.logger.debug "Looking for merged_workers in #{worker.class.name}: #{merged_workers.inspect}"
|
369
|
-
ret = {}
|
370
|
-
if merged_workers && merged_workers.size > 0
|
371
|
-
merged_workers.each_pair do |k, mw|
|
372
|
-
IronWorker.logger.debug "merged worker found in #{worker.class.name}: #{mw.inspect}"
|
373
|
-
ret[mw[:name]] = mw
|
374
|
-
end
|
375
|
-
end
|
376
|
-
ret
|
377
|
-
end
|
378
|
-
|
379
|
-
def upload_if_needed(options={})
|
380
|
-
return if is_remote?
|
381
|
-
check_service
|
382
|
-
IronWorker.service.check_config
|
383
|
-
|
384
|
-
before_upload
|
385
|
-
|
386
|
-
merged = self.class.instance_variable_get(:@merged)
|
387
|
-
|
388
|
-
# do merged_workers first because we need to get their subclasses and what not too
|
389
|
-
merged_workers = self.class.instance_variable_get(:@merged_workers)
|
390
|
-
if merged_workers && merged_workers.size > 0
|
391
|
-
IronWorker.logger.debug 'now uploading merged workers ' + merged_workers.inspect
|
392
|
-
merged_workers.each_pair do |mw, v|
|
393
|
-
IronWorker.logger.debug 'Instantiating and uploading ' + v.inspect
|
394
|
-
mw_instantiated = Kernel.const_get(v[:class_name]).new
|
395
|
-
mw_instantiated.upload
|
396
|
-
|
397
|
-
merged, rfile, subclass = IronWorker::Base.extract_superclasses_merges(mw_instantiated, merged)
|
398
|
-
merged.merge!(IronWorker::Base.extract_merged_workers(mw_instantiated))
|
399
|
-
|
400
|
-
end
|
401
|
-
end
|
402
|
-
|
403
|
-
if !uploaded?
|
404
|
-
unmerged = self.class.instance_variable_get(:@unmerged)
|
405
|
-
merged_gems = self.class.instance_variable_get(:@merged_gems)
|
406
|
-
unmerged_gems = self.class.instance_variable_get(:@unmerged_gems)
|
407
|
-
merged_mailers = self.class.instance_variable_get(:@merged_mailers)
|
408
|
-
merged_folders = self.class.instance_variable_get(:@merged_folders)
|
409
|
-
merged, rfile, subclass = IronWorker::Base.extract_superclasses_merges(self, merged)
|
410
|
-
merged_mailers = merged_mailers.merge(IronWorker.config.mailers) if IronWorker.config.mailers
|
411
|
-
unless merged_gems.size == 0
|
412
|
-
merged_gems = gems_to_merge(merged_gems)
|
413
|
-
end
|
414
|
-
|
415
|
-
options_for_upload = {:merge=>merged, :unmerge=>unmerged, :merged_gems=>merged_gems, :unmerged_gems=>unmerged_gems, :merged_mailers=>merged_mailers, :merged_folders=>merged_folders}
|
416
|
-
options_for_upload.merge!(options)
|
417
|
-
IronWorker.service.upload(rfile, subclass.name, options_for_upload)
|
418
|
-
self.class.instance_variable_set(:@uploaded, true)
|
419
|
-
else
|
420
|
-
IronWorker.logger.debug 'Already uploaded for ' + self.class.name
|
421
|
-
end
|
422
|
-
|
423
|
-
after_upload
|
424
|
-
end
|
425
|
-
|
426
|
-
def sw_get_data
|
427
|
-
data = {}
|
428
|
-
|
429
|
-
payload = {}
|
430
|
-
# todo: should put these down a layer, eg: payload[:attributes]
|
431
|
-
self.instance_variables.each do |iv|
|
432
|
-
payload[iv] = instance_variable_get(iv)
|
433
|
-
end
|
434
|
-
data['class_name'] = self.class.name
|
435
|
-
data[:attr_encoded] = Base64.encode64(payload.to_json)
|
436
|
-
data[:file_name] = File.basename(self.class.instance_variable_get(:@caller_file))
|
437
|
-
if defined?(Rails)
|
438
|
-
data[:rails] = {}
|
439
|
-
data[:rails]['env'] = Rails.env
|
440
|
-
data[:rails]['version'] = Rails.version
|
441
|
-
end
|
442
|
-
config_data = IronWorker.config.get_atts_to_send
|
443
|
-
data[:sw_config] = config_data
|
444
|
-
return data
|
445
|
-
end
|
446
|
-
|
447
|
-
|
448
|
-
end
|
449
|
-
end
|