iron_worker 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +72 -0
- data/VERSION.yml +5 -0
- data/lib/generators/iron_worker/iron_worker_generator.rb +13 -0
- data/lib/iron_worker.rb +49 -0
- data/lib/iron_worker/api.rb +242 -0
- data/lib/iron_worker/base.rb +432 -0
- data/lib/iron_worker/config.rb +265 -0
- data/lib/iron_worker/rails2_init.rb +8 -0
- data/lib/iron_worker/railtie.rb +16 -0
- data/lib/iron_worker/server/overrides.rb +199 -0
- data/lib/iron_worker/server/runner.rb +80 -0
- data/lib/iron_worker/service.rb +553 -0
- data/lib/iron_worker/uber_client.rb +117 -0
- data/lib/iron_worker/used_in_worker.rb +11 -0
- data/lib/iron_worker/utils.rb +11 -0
- data/rails/init.rb +0 -0
- metadata +105 -0
@@ -0,0 +1,265 @@
|
|
1
|
+
module IronWorker
|
2
|
+
|
3
|
+
|
4
|
+
# Config is used to setup the IronWorker client.
|
5
|
+
# You must set the access_key and secret_key.
|
6
|
+
#
|
7
|
+
# config.global_attributes allows you to specify attributes that will automatically be set on every worker,
|
8
|
+
# this is good for database connection information or things that will be used across the board.
|
9
|
+
#
|
10
|
+
# config.database configures a database connection. If specified like ActiveRecord, IronWorker will automatically establish a connection
|
11
|
+
# for you before running your worker.
|
12
|
+
class Config
|
13
|
+
attr_accessor :token,
|
14
|
+
:project_id,
|
15
|
+
:scheme,
|
16
|
+
:host,
|
17
|
+
:port,
|
18
|
+
:global_attributes,
|
19
|
+
:models,
|
20
|
+
:mailers,
|
21
|
+
#:gems, # todo: move anything that uses this to merged_gems
|
22
|
+
:database,
|
23
|
+
:mailer,
|
24
|
+
:extra_requires,
|
25
|
+
#:auto_merge,
|
26
|
+
:server_gems,
|
27
|
+
:merged,
|
28
|
+
:unmerged,
|
29
|
+
:merged_gems,
|
30
|
+
:unmerged_gems,
|
31
|
+
:force_upload
|
32
|
+
|
33
|
+
|
34
|
+
def initialize
|
35
|
+
@global_attributes = {}
|
36
|
+
@extra_requires = []
|
37
|
+
@merged = {}
|
38
|
+
@unmerged = {}
|
39
|
+
@merged_gems = {}
|
40
|
+
@unmerged_gems = {}
|
41
|
+
@mailers = {}
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
def access_key=(x)
|
46
|
+
raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
|
47
|
+
end
|
48
|
+
def secret_key=(x)
|
49
|
+
raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
|
50
|
+
end
|
51
|
+
|
52
|
+
@gems_to_skip = ['actionmailer', 'actionpack', 'activemodel', 'activeresource', 'activesupport',
|
53
|
+
'bundler',
|
54
|
+
'mail',
|
55
|
+
'mysql2',
|
56
|
+
'rails',
|
57
|
+
'tzinfo' # HUGE!
|
58
|
+
]
|
59
|
+
|
60
|
+
def self.gems_to_skip
|
61
|
+
@gems_to_skip
|
62
|
+
end
|
63
|
+
|
64
|
+
def auto_merge=(b)
|
65
|
+
if b
|
66
|
+
IronWorker.logger.info "Initializing IronWorker for Rails 3..."
|
67
|
+
start_time = Time.now
|
68
|
+
IronWorker.configure do |c2|
|
69
|
+
models_path = File.join(Rails.root, 'app/models/*.rb')
|
70
|
+
models = Dir.glob(models_path)
|
71
|
+
c2.models = models
|
72
|
+
models.each { |model| c2.merge(model) }
|
73
|
+
mailers_path = File.join(Rails.root, 'app/mailers/*.rb')
|
74
|
+
Dir.glob(mailers_path).collect { |m| c2.mailers[File.basename(m)] = {:filename=>m, :name => File.basename(m), :path_to_templates=>File.join(Rails.root, "app/views/#{File.basename(m, File.extname(m))}")} }
|
75
|
+
c2.extra_requires += ['active_support/core_ext', 'action_mailer']
|
76
|
+
#puts 'DB FILE=' + File.join(Rails.root, 'config', 'database.yml').to_s
|
77
|
+
if defined?(ActiveRecord) && File.exist?(File.join(Rails.root, 'config', 'database.yml'))
|
78
|
+
c2.extra_requires += ['active_record']
|
79
|
+
c2.database = Rails.configuration.database_configuration[Rails.env]
|
80
|
+
else
|
81
|
+
#puts 'NOT DOING ACTIVERECORD'
|
82
|
+
end
|
83
|
+
|
84
|
+
if defined?(ActionMailer) && ActionMailer::Base.smtp_settings
|
85
|
+
c2.mailer = ActionMailer::Base.smtp_settings
|
86
|
+
end
|
87
|
+
c2.merged_gems.merge!(get_required_gems) if defined?(Bundler)
|
88
|
+
IronWorker.logger.debug "MODELS " + c2.models.inspect
|
89
|
+
IronWorker.logger.debug "MAILERS " + c2.mailers.inspect
|
90
|
+
IronWorker.logger.debug "DATABASE " + c2.database.inspect
|
91
|
+
#IronWorker.logger.debug "GEMS " + c2.gems.inspect
|
92
|
+
end
|
93
|
+
end_time = Time.now
|
94
|
+
IronWorker.logger.info "IronWorker initialized. Duration: #{((end_time.to_f-start_time.to_f) * 1000.0).to_i} ms"
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
def get_required_gems
|
100
|
+
gems_in_gemfile = Bundler.environment.dependencies.select { |d| d.groups.include?(:default) }
|
101
|
+
IronWorker.logger.debug 'gems in gemfile=' + gems_in_gemfile.inspect
|
102
|
+
gems = {}
|
103
|
+
specs = Bundler.load.specs
|
104
|
+
IronWorker.logger.debug 'Bundler specs=' + specs.inspect
|
105
|
+
IronWorker.logger.debug "gems_to_skip=" + self.class.gems_to_skip.inspect
|
106
|
+
specs.each do |spec|
|
107
|
+
IronWorker.logger.debug 'spec.name=' + spec.name.inspect
|
108
|
+
IronWorker.logger.debug 'spec=' + spec.inspect
|
109
|
+
if self.class.gems_to_skip.include?(spec.name)
|
110
|
+
IronWorker.logger.debug "Skipping #{spec.name}"
|
111
|
+
next
|
112
|
+
end
|
113
|
+
# next if dep.name=='rails' #monkey patch
|
114
|
+
gem_info = {:name=>spec.name, :version=>spec.version}
|
115
|
+
gem_info[:auto_merged] = true
|
116
|
+
gem_info[:merge] = true
|
117
|
+
# Now find dependency in gemfile in case user set the require
|
118
|
+
dep = gems_in_gemfile.find { |g| g.name == gem_info[:name] }
|
119
|
+
if dep
|
120
|
+
IronWorker.logger.debug 'dep found in gemfile: ' + dep.inspect
|
121
|
+
IronWorker.logger.debug 'autorequire=' + dep.autorequire.inspect
|
122
|
+
gem_info[:require] = dep.autorequire if dep.autorequire
|
123
|
+
# spec = specs.find { |g| g.name==gem_info[:name] }
|
124
|
+
end
|
125
|
+
gem_info[:version] = spec.version.to_s
|
126
|
+
gems[gem_info[:name]] = gem_info
|
127
|
+
path = IronWorker::Service.get_gem_path(gem_info)
|
128
|
+
if path
|
129
|
+
gem_info[:path] = path
|
130
|
+
if gem_info[:require].nil? && dep
|
131
|
+
# see if we should try to require this in our worker
|
132
|
+
require_path = gem_info[:path] + "/lib/#{gem_info[:name]}.rb"
|
133
|
+
IronWorker.logger.debug "require_path=" + require_path
|
134
|
+
if File.exists?(require_path)
|
135
|
+
IronWorker.logger.debug "File exists for require"
|
136
|
+
gem_info[:require] = gem_info[:name]
|
137
|
+
else
|
138
|
+
IronWorker.logger.debug "no require"
|
139
|
+
# gem_info[:no_require] = true
|
140
|
+
end
|
141
|
+
end
|
142
|
+
else
|
143
|
+
IronWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
|
144
|
+
end
|
145
|
+
# else
|
146
|
+
# IronWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
|
147
|
+
# raise "Could not find gem spec for #{gem_info[:name]}"
|
148
|
+
# end
|
149
|
+
end
|
150
|
+
gems
|
151
|
+
end
|
152
|
+
|
153
|
+
def get_server_gems
|
154
|
+
return []
|
155
|
+
# skipping this now, don't want any server dependencies if possible
|
156
|
+
self.server_gems = IronWorker.service.get_server_gems unless self.server_gems
|
157
|
+
self.server_gems
|
158
|
+
end
|
159
|
+
|
160
|
+
def get_atts_to_send
|
161
|
+
config_data = {}
|
162
|
+
config_data['token'] = token
|
163
|
+
config_data['project_id'] = project_id
|
164
|
+
config_data['database'] = self.database if self.database
|
165
|
+
config_data['mailer'] = self.mailer if self.mailer
|
166
|
+
config_data['global_attributes'] = self.global_attributes if self.global_attributes
|
167
|
+
config_data['scheme'] = self.scheme if self.scheme
|
168
|
+
config_data['host'] = self.host if self.host
|
169
|
+
config_data['port'] = self.port if self.port
|
170
|
+
config_data
|
171
|
+
end
|
172
|
+
|
173
|
+
def merge(file)
|
174
|
+
f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
|
175
|
+
fbase = f2[:basename]
|
176
|
+
ret = f2
|
177
|
+
@merged[fbase] = ret
|
178
|
+
ret
|
179
|
+
end
|
180
|
+
|
181
|
+
def unmerge(file)
|
182
|
+
f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
|
183
|
+
fbase = f2[:basename]
|
184
|
+
@unmerged[fbase] =f2
|
185
|
+
@merged.delete(fbase)
|
186
|
+
end
|
187
|
+
|
188
|
+
# Merge a gem globally here
|
189
|
+
def merge_gem(gem_name, options={})
|
190
|
+
merged_gems[gem_name.to_s] = IronWorker::MergeHelper.create_gem_info(gem_name, options)
|
191
|
+
end
|
192
|
+
|
193
|
+
# Unmerge a global gem
|
194
|
+
def unmerge_gem(gem_name)
|
195
|
+
gs = gem_name.to_s
|
196
|
+
gem_info = {:name=>gs}
|
197
|
+
unmerged_gems[gs] = gem_info
|
198
|
+
merged_gems.delete(gs)
|
199
|
+
end
|
200
|
+
|
201
|
+
end
|
202
|
+
|
203
|
+
|
204
|
+
class MergeHelper
|
205
|
+
|
206
|
+
# callerr is original file that is calling the merge function, ie: your worker.
|
207
|
+
# See Base for examples.
|
208
|
+
def self.check_for_file(f, callerr)
|
209
|
+
IronWorker.logger.debug 'Checking for ' + f.to_s
|
210
|
+
f = f.to_str
|
211
|
+
f_ext = File.extname(f)
|
212
|
+
if f_ext.empty?
|
213
|
+
f_ext = ".rb"
|
214
|
+
f << f_ext
|
215
|
+
end
|
216
|
+
exists = false
|
217
|
+
if File.exist? f
|
218
|
+
exists = true
|
219
|
+
else
|
220
|
+
# try relative
|
221
|
+
# p caller
|
222
|
+
f2 = File.join(File.dirname(callerr), f)
|
223
|
+
puts 'f2=' + f2
|
224
|
+
if File.exist? f2
|
225
|
+
exists = true
|
226
|
+
f = f2
|
227
|
+
end
|
228
|
+
end
|
229
|
+
unless exists
|
230
|
+
raise "File not found: " + f
|
231
|
+
end
|
232
|
+
f = File.expand_path(f)
|
233
|
+
require f if f_ext == '.rb'
|
234
|
+
ret = {}
|
235
|
+
ret[:path] = f
|
236
|
+
ret[:extname] = f_ext
|
237
|
+
ret[:basename] = File.basename(f)
|
238
|
+
ret[:name] = ret[:basename]
|
239
|
+
ret
|
240
|
+
end
|
241
|
+
|
242
|
+
def self.create_gem_info(gem_name, options={})
|
243
|
+
gem_info = {:name=>gem_name, :merge=>true}
|
244
|
+
if options.is_a?(Hash)
|
245
|
+
gem_info.merge!(options)
|
246
|
+
if options[:include_dirs]
|
247
|
+
gem_info[:include_dirs] = options[:include_dirs].is_a?(Array) ? options[:include_dirs] : [options[:include_dirs]]
|
248
|
+
end
|
249
|
+
else
|
250
|
+
gem_info[:version] = options
|
251
|
+
end
|
252
|
+
gem_info[:require] ||= gem_name
|
253
|
+
|
254
|
+
path = IronWorker::Service.get_gem_path(gem_info)
|
255
|
+
IronWorker.logger.debug "Gem path=#{path}"
|
256
|
+
if !path
|
257
|
+
raise "Gem '#{gem_name}' not found."
|
258
|
+
end
|
259
|
+
gem_info[:path] = path
|
260
|
+
gem_info
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
end
|
265
|
+
|
@@ -0,0 +1,8 @@
|
|
1
|
+
#puts "Initializing list of Rails models..."
|
2
|
+
IronWorker.configure do |config|
|
3
|
+
path = File.join(Rails.root, 'app/models/*.rb')
|
4
|
+
# puts 'path=' + path
|
5
|
+
config.models = Dir.glob(path)
|
6
|
+
config.extra_requires += ['active_support/core_ext', 'active_record', 'action_mailer']
|
7
|
+
# puts 'config.models=' + config.models.inspect
|
8
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# see http://api.rubyonrails.org/classes/Rails/Railtie.html
|
2
|
+
|
3
|
+
require 'iron_worker'
|
4
|
+
require 'rails'
|
5
|
+
|
6
|
+
module IronWorker
|
7
|
+
class Railtie < Rails::Railtie
|
8
|
+
|
9
|
+
|
10
|
+
initializer "iron_worker.configure_rails_initialization" do |app|
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,199 @@
|
|
1
|
+
# This is used when a bad worker is uploaded.
|
2
|
+
|
3
|
+
module IronWorker
|
4
|
+
|
5
|
+
class InvalidWorkerError < StandardError; end
|
6
|
+
|
7
|
+
class << self
|
8
|
+
def running_class=(rc)
|
9
|
+
@running_class = rc
|
10
|
+
end
|
11
|
+
def running_class
|
12
|
+
@running_class
|
13
|
+
end
|
14
|
+
def task_data=(td)
|
15
|
+
@task_data = td
|
16
|
+
end
|
17
|
+
def task_data
|
18
|
+
@task_data
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.disable_queueing()
|
23
|
+
@queueing_enabled = false
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.enable_queueing()
|
27
|
+
@queueing_enabled = true
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.queueing_enabled?
|
31
|
+
@queueing_enabled
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
class Base
|
36
|
+
|
37
|
+
class << self
|
38
|
+
|
39
|
+
def merge(*files)
|
40
|
+
#files.each do |file|
|
41
|
+
# file = file.to_s
|
42
|
+
# unless file.end_with?(".rb")
|
43
|
+
# file << ".rb"
|
44
|
+
# end
|
45
|
+
# #puts 'code_dir=' + code_dir.inspect
|
46
|
+
# filename = File.join(code_dir, File.basename(file))
|
47
|
+
# #puts "merge #{filename}"
|
48
|
+
# #puts "FILENAME #{filename}"
|
49
|
+
# require filename if File.exist?(filename) # for backwards compatability
|
50
|
+
#end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
def merge_folder(path)
|
56
|
+
#puts "PATH=#{path}"
|
57
|
+
#puts "#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"
|
58
|
+
#Dir["#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"].each do |f|
|
59
|
+
# puts "requiring #{f.inspect}"
|
60
|
+
# require f if File.exist?(f)
|
61
|
+
#end
|
62
|
+
end
|
63
|
+
|
64
|
+
def unmerge(*files)
|
65
|
+
# ignore this here
|
66
|
+
end
|
67
|
+
|
68
|
+
def merge_mailer(mailer, params=nil)
|
69
|
+
#merge(mailer)
|
70
|
+
end
|
71
|
+
|
72
|
+
def merge_gem(gem, version=nil)
|
73
|
+
#gem_info = {:name=>gem}
|
74
|
+
#if version.is_a?(Hash)
|
75
|
+
# gem_info.merge!(version)
|
76
|
+
#else
|
77
|
+
# gem_info[:version] = version
|
78
|
+
#end
|
79
|
+
#gem_name =(gem.match(/^[a-zA-Z0-9\-_]+/)[0])
|
80
|
+
#$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}") #backwards compatibility, should be removed later
|
81
|
+
#$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}/lib")
|
82
|
+
# # what's the diff here? This one seems more common: $:.unshift File.join(File.dirname(__FILE__), "/gems/#{gem_name}")
|
83
|
+
# #puts 'gem_info = ' + gem_info.inspect
|
84
|
+
#require gem_info[:require] || gem_name
|
85
|
+
end
|
86
|
+
|
87
|
+
def merge_worker(file, class_name)
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
def is_remote?
|
94
|
+
true
|
95
|
+
end
|
96
|
+
|
97
|
+
#:todo remove this method later, when new iron_worker gem will be released
|
98
|
+
def is_local?
|
99
|
+
!is_remote?
|
100
|
+
end
|
101
|
+
|
102
|
+
def log(str)
|
103
|
+
puts str.to_s
|
104
|
+
end
|
105
|
+
|
106
|
+
def set_progress(hash)
|
107
|
+
#puts 'set_progress self=' + self.inspect
|
108
|
+
IronWorker.service.set_progress(self.task_id, hash)
|
109
|
+
end
|
110
|
+
|
111
|
+
def something
|
112
|
+
puts 'which class? ' + self.class.name
|
113
|
+
end
|
114
|
+
|
115
|
+
def user_dir
|
116
|
+
# puts 'user_dir=' + @context.user_dir.to_s
|
117
|
+
@user_dir || "./"
|
118
|
+
end
|
119
|
+
|
120
|
+
def sw_set_data(data)
|
121
|
+
if data["attr_encoded"]
|
122
|
+
# new way, attributes are base 64 encoded
|
123
|
+
data = JSON.parse(Base64.decode64(data["attr_encoded"]))
|
124
|
+
end
|
125
|
+
|
126
|
+
data.each_pair do |k, v|
|
127
|
+
next unless k[0] == "@"
|
128
|
+
# puts "setting instance_variable #{k}=#{v}"
|
129
|
+
self.instance_variable_set(k, v)
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
def upload_if_needed(options={})
|
135
|
+
puts "No uploading in worker service."
|
136
|
+
end
|
137
|
+
|
138
|
+
alias_method :orig_queue, :queue
|
139
|
+
alias_method :orig_schedule, :schedule
|
140
|
+
alias_method :orig_status, :status
|
141
|
+
|
142
|
+
def queue(options={})
|
143
|
+
if IronWorker.queueing_enabled? && (!same_clazz? || options[:recursive])
|
144
|
+
orig_queue(options)
|
145
|
+
# data = sw_get_data()
|
146
|
+
# queue_other(self.class.name, data)
|
147
|
+
else
|
148
|
+
log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in queueing, pass in :recursive=>true to bypass this." : "Queuing disabled while loading.")
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def schedule(schedule)
|
153
|
+
if IronWorker.queueing_enabled? && (!same_clazz? || schedule[:recursive])
|
154
|
+
orig_schedule(schedule)
|
155
|
+
# data = sw_get_data()
|
156
|
+
# schedule_other(self.class.name, data, schedule)
|
157
|
+
else
|
158
|
+
log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in scheduling." : "Scheduling disabled while loading.")
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
def status
|
164
|
+
if IronWorker.queueing_enabled?
|
165
|
+
orig_status
|
166
|
+
else
|
167
|
+
log "Status disabled while loading."
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def same_clazz?
|
172
|
+
IronWorker.running_class == self.class
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
class Service < IronWorker::Api::Client
|
179
|
+
def upload(filename, class_name, options={})
|
180
|
+
#puts "Skipping upload, We don't upload from run.rb!"
|
181
|
+
# don't upload, should already be here.
|
182
|
+
end
|
183
|
+
|
184
|
+
def add_sw_params(hash_to_send)
|
185
|
+
hash_to_send["token"] = self.config.token
|
186
|
+
hash_to_send["project_id"] = self.config.project_id
|
187
|
+
hash_to_send["api_version"] = IronWorker.api_version
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
|
192
|
+
module UsedInWorker
|
193
|
+
def log(str)
|
194
|
+
puts str.to_s
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
|
199
|
+
end
|