iron_worker 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,265 @@
1
+ module IronWorker
2
+
3
+
4
+ # Config is used to setup the IronWorker client.
5
+ # You must set the access_key and secret_key.
6
+ #
7
+ # config.global_attributes allows you to specify attributes that will automatically be set on every worker,
8
+ # this is good for database connection information or things that will be used across the board.
9
+ #
10
+ # config.database configures a database connection. If specified like ActiveRecord, IronWorker will automatically establish a connection
11
+ # for you before running your worker.
12
+ class Config
13
+ attr_accessor :token,
14
+ :project_id,
15
+ :scheme,
16
+ :host,
17
+ :port,
18
+ :global_attributes,
19
+ :models,
20
+ :mailers,
21
+ #:gems, # todo: move anything that uses this to merged_gems
22
+ :database,
23
+ :mailer,
24
+ :extra_requires,
25
+ #:auto_merge,
26
+ :server_gems,
27
+ :merged,
28
+ :unmerged,
29
+ :merged_gems,
30
+ :unmerged_gems,
31
+ :force_upload
32
+
33
+
34
+ def initialize
35
+ @global_attributes = {}
36
+ @extra_requires = []
37
+ @merged = {}
38
+ @unmerged = {}
39
+ @merged_gems = {}
40
+ @unmerged_gems = {}
41
+ @mailers = {}
42
+
43
+ end
44
+
45
+ def access_key=(x)
46
+ raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
47
+ end
48
+ def secret_key=(x)
49
+ raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
50
+ end
51
+
52
+ @gems_to_skip = ['actionmailer', 'actionpack', 'activemodel', 'activeresource', 'activesupport',
53
+ 'bundler',
54
+ 'mail',
55
+ 'mysql2',
56
+ 'rails',
57
+ 'tzinfo' # HUGE!
58
+ ]
59
+
60
+ def self.gems_to_skip
61
+ @gems_to_skip
62
+ end
63
+
64
+ def auto_merge=(b)
65
+ if b
66
+ IronWorker.logger.info "Initializing IronWorker for Rails 3..."
67
+ start_time = Time.now
68
+ IronWorker.configure do |c2|
69
+ models_path = File.join(Rails.root, 'app/models/*.rb')
70
+ models = Dir.glob(models_path)
71
+ c2.models = models
72
+ models.each { |model| c2.merge(model) }
73
+ mailers_path = File.join(Rails.root, 'app/mailers/*.rb')
74
+ Dir.glob(mailers_path).collect { |m| c2.mailers[File.basename(m)] = {:filename=>m, :name => File.basename(m), :path_to_templates=>File.join(Rails.root, "app/views/#{File.basename(m, File.extname(m))}")} }
75
+ c2.extra_requires += ['active_support/core_ext', 'action_mailer']
76
+ #puts 'DB FILE=' + File.join(Rails.root, 'config', 'database.yml').to_s
77
+ if defined?(ActiveRecord) && File.exist?(File.join(Rails.root, 'config', 'database.yml'))
78
+ c2.extra_requires += ['active_record']
79
+ c2.database = Rails.configuration.database_configuration[Rails.env]
80
+ else
81
+ #puts 'NOT DOING ACTIVERECORD'
82
+ end
83
+
84
+ if defined?(ActionMailer) && ActionMailer::Base.smtp_settings
85
+ c2.mailer = ActionMailer::Base.smtp_settings
86
+ end
87
+ c2.merged_gems.merge!(get_required_gems) if defined?(Bundler)
88
+ IronWorker.logger.debug "MODELS " + c2.models.inspect
89
+ IronWorker.logger.debug "MAILERS " + c2.mailers.inspect
90
+ IronWorker.logger.debug "DATABASE " + c2.database.inspect
91
+ #IronWorker.logger.debug "GEMS " + c2.gems.inspect
92
+ end
93
+ end_time = Time.now
94
+ IronWorker.logger.info "IronWorker initialized. Duration: #{((end_time.to_f-start_time.to_f) * 1000.0).to_i} ms"
95
+ end
96
+ end
97
+
98
+
99
+ def get_required_gems
100
+ gems_in_gemfile = Bundler.environment.dependencies.select { |d| d.groups.include?(:default) }
101
+ IronWorker.logger.debug 'gems in gemfile=' + gems_in_gemfile.inspect
102
+ gems = {}
103
+ specs = Bundler.load.specs
104
+ IronWorker.logger.debug 'Bundler specs=' + specs.inspect
105
+ IronWorker.logger.debug "gems_to_skip=" + self.class.gems_to_skip.inspect
106
+ specs.each do |spec|
107
+ IronWorker.logger.debug 'spec.name=' + spec.name.inspect
108
+ IronWorker.logger.debug 'spec=' + spec.inspect
109
+ if self.class.gems_to_skip.include?(spec.name)
110
+ IronWorker.logger.debug "Skipping #{spec.name}"
111
+ next
112
+ end
113
+ # next if dep.name=='rails' #monkey patch
114
+ gem_info = {:name=>spec.name, :version=>spec.version}
115
+ gem_info[:auto_merged] = true
116
+ gem_info[:merge] = true
117
+ # Now find dependency in gemfile in case user set the require
118
+ dep = gems_in_gemfile.find { |g| g.name == gem_info[:name] }
119
+ if dep
120
+ IronWorker.logger.debug 'dep found in gemfile: ' + dep.inspect
121
+ IronWorker.logger.debug 'autorequire=' + dep.autorequire.inspect
122
+ gem_info[:require] = dep.autorequire if dep.autorequire
123
+ # spec = specs.find { |g| g.name==gem_info[:name] }
124
+ end
125
+ gem_info[:version] = spec.version.to_s
126
+ gems[gem_info[:name]] = gem_info
127
+ path = IronWorker::Service.get_gem_path(gem_info)
128
+ if path
129
+ gem_info[:path] = path
130
+ if gem_info[:require].nil? && dep
131
+ # see if we should try to require this in our worker
132
+ require_path = gem_info[:path] + "/lib/#{gem_info[:name]}.rb"
133
+ IronWorker.logger.debug "require_path=" + require_path
134
+ if File.exists?(require_path)
135
+ IronWorker.logger.debug "File exists for require"
136
+ gem_info[:require] = gem_info[:name]
137
+ else
138
+ IronWorker.logger.debug "no require"
139
+ # gem_info[:no_require] = true
140
+ end
141
+ end
142
+ else
143
+ IronWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
144
+ end
145
+ # else
146
+ # IronWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
147
+ # raise "Could not find gem spec for #{gem_info[:name]}"
148
+ # end
149
+ end
150
+ gems
151
+ end
152
+
153
+ def get_server_gems
154
+ return []
155
+ # skipping this now, don't want any server dependencies if possible
156
+ self.server_gems = IronWorker.service.get_server_gems unless self.server_gems
157
+ self.server_gems
158
+ end
159
+
160
+ def get_atts_to_send
161
+ config_data = {}
162
+ config_data['token'] = token
163
+ config_data['project_id'] = project_id
164
+ config_data['database'] = self.database if self.database
165
+ config_data['mailer'] = self.mailer if self.mailer
166
+ config_data['global_attributes'] = self.global_attributes if self.global_attributes
167
+ config_data['scheme'] = self.scheme if self.scheme
168
+ config_data['host'] = self.host if self.host
169
+ config_data['port'] = self.port if self.port
170
+ config_data
171
+ end
172
+
173
+ def merge(file)
174
+ f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
175
+ fbase = f2[:basename]
176
+ ret = f2
177
+ @merged[fbase] = ret
178
+ ret
179
+ end
180
+
181
+ def unmerge(file)
182
+ f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
183
+ fbase = f2[:basename]
184
+ @unmerged[fbase] =f2
185
+ @merged.delete(fbase)
186
+ end
187
+
188
+ # Merge a gem globally here
189
+ def merge_gem(gem_name, options={})
190
+ merged_gems[gem_name.to_s] = IronWorker::MergeHelper.create_gem_info(gem_name, options)
191
+ end
192
+
193
+ # Unmerge a global gem
194
+ def unmerge_gem(gem_name)
195
+ gs = gem_name.to_s
196
+ gem_info = {:name=>gs}
197
+ unmerged_gems[gs] = gem_info
198
+ merged_gems.delete(gs)
199
+ end
200
+
201
+ end
202
+
203
+
204
+ class MergeHelper
205
+
206
+ # callerr is original file that is calling the merge function, ie: your worker.
207
+ # See Base for examples.
208
+ def self.check_for_file(f, callerr)
209
+ IronWorker.logger.debug 'Checking for ' + f.to_s
210
+ f = f.to_str
211
+ f_ext = File.extname(f)
212
+ if f_ext.empty?
213
+ f_ext = ".rb"
214
+ f << f_ext
215
+ end
216
+ exists = false
217
+ if File.exist? f
218
+ exists = true
219
+ else
220
+ # try relative
221
+ # p caller
222
+ f2 = File.join(File.dirname(callerr), f)
223
+ puts 'f2=' + f2
224
+ if File.exist? f2
225
+ exists = true
226
+ f = f2
227
+ end
228
+ end
229
+ unless exists
230
+ raise "File not found: " + f
231
+ end
232
+ f = File.expand_path(f)
233
+ require f if f_ext == '.rb'
234
+ ret = {}
235
+ ret[:path] = f
236
+ ret[:extname] = f_ext
237
+ ret[:basename] = File.basename(f)
238
+ ret[:name] = ret[:basename]
239
+ ret
240
+ end
241
+
242
+ def self.create_gem_info(gem_name, options={})
243
+ gem_info = {:name=>gem_name, :merge=>true}
244
+ if options.is_a?(Hash)
245
+ gem_info.merge!(options)
246
+ if options[:include_dirs]
247
+ gem_info[:include_dirs] = options[:include_dirs].is_a?(Array) ? options[:include_dirs] : [options[:include_dirs]]
248
+ end
249
+ else
250
+ gem_info[:version] = options
251
+ end
252
+ gem_info[:require] ||= gem_name
253
+
254
+ path = IronWorker::Service.get_gem_path(gem_info)
255
+ IronWorker.logger.debug "Gem path=#{path}"
256
+ if !path
257
+ raise "Gem '#{gem_name}' not found."
258
+ end
259
+ gem_info[:path] = path
260
+ gem_info
261
+ end
262
+ end
263
+
264
+ end
265
+
@@ -0,0 +1,8 @@
1
+ #puts "Initializing list of Rails models..."
2
+ IronWorker.configure do |config|
3
+ path = File.join(Rails.root, 'app/models/*.rb')
4
+ # puts 'path=' + path
5
+ config.models = Dir.glob(path)
6
+ config.extra_requires += ['active_support/core_ext', 'active_record', 'action_mailer']
7
+ # puts 'config.models=' + config.models.inspect
8
+ end
@@ -0,0 +1,16 @@
1
+ # see http://api.rubyonrails.org/classes/Rails/Railtie.html
2
+
3
+ require 'iron_worker'
4
+ require 'rails'
5
+
6
+ module IronWorker
7
+ class Railtie < Rails::Railtie
8
+
9
+
10
+ initializer "iron_worker.configure_rails_initialization" do |app|
11
+
12
+ end
13
+
14
+
15
+ end
16
+ end
@@ -0,0 +1,199 @@
1
+ # This is used when a bad worker is uploaded.
2
+
3
+ module IronWorker
4
+
5
+ class InvalidWorkerError < StandardError; end
6
+
7
+ class << self
8
+ def running_class=(rc)
9
+ @running_class = rc
10
+ end
11
+ def running_class
12
+ @running_class
13
+ end
14
+ def task_data=(td)
15
+ @task_data = td
16
+ end
17
+ def task_data
18
+ @task_data
19
+ end
20
+ end
21
+
22
+ def self.disable_queueing()
23
+ @queueing_enabled = false
24
+ end
25
+
26
+ def self.enable_queueing()
27
+ @queueing_enabled = true
28
+ end
29
+
30
+ def self.queueing_enabled?
31
+ @queueing_enabled
32
+ end
33
+
34
+
35
+ class Base
36
+
37
+ class << self
38
+
39
+ def merge(*files)
40
+ #files.each do |file|
41
+ # file = file.to_s
42
+ # unless file.end_with?(".rb")
43
+ # file << ".rb"
44
+ # end
45
+ # #puts 'code_dir=' + code_dir.inspect
46
+ # filename = File.join(code_dir, File.basename(file))
47
+ # #puts "merge #{filename}"
48
+ # #puts "FILENAME #{filename}"
49
+ # require filename if File.exist?(filename) # for backwards compatability
50
+ #end
51
+
52
+ end
53
+
54
+
55
+ def merge_folder(path)
56
+ #puts "PATH=#{path}"
57
+ #puts "#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"
58
+ #Dir["#{code_dir}/#{Digest::MD5.hexdigest(path)}/**/*.rb"].each do |f|
59
+ # puts "requiring #{f.inspect}"
60
+ # require f if File.exist?(f)
61
+ #end
62
+ end
63
+
64
+ def unmerge(*files)
65
+ # ignore this here
66
+ end
67
+
68
+ def merge_mailer(mailer, params=nil)
69
+ #merge(mailer)
70
+ end
71
+
72
+ def merge_gem(gem, version=nil)
73
+ #gem_info = {:name=>gem}
74
+ #if version.is_a?(Hash)
75
+ # gem_info.merge!(version)
76
+ #else
77
+ # gem_info[:version] = version
78
+ #end
79
+ #gem_name =(gem.match(/^[a-zA-Z0-9\-_]+/)[0])
80
+ #$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}") #backwards compatibility, should be removed later
81
+ #$LOAD_PATH << File.join(code_dir, "/gems/#{gem_name}/lib")
82
+ # # what's the diff here? This one seems more common: $:.unshift File.join(File.dirname(__FILE__), "/gems/#{gem_name}")
83
+ # #puts 'gem_info = ' + gem_info.inspect
84
+ #require gem_info[:require] || gem_name
85
+ end
86
+
87
+ def merge_worker(file, class_name)
88
+
89
+ end
90
+
91
+ end
92
+
93
+ def is_remote?
94
+ true
95
+ end
96
+
97
+ #:todo remove this method later, when new iron_worker gem will be released
98
+ def is_local?
99
+ !is_remote?
100
+ end
101
+
102
+ def log(str)
103
+ puts str.to_s
104
+ end
105
+
106
+ def set_progress(hash)
107
+ #puts 'set_progress self=' + self.inspect
108
+ IronWorker.service.set_progress(self.task_id, hash)
109
+ end
110
+
111
+ def something
112
+ puts 'which class? ' + self.class.name
113
+ end
114
+
115
+ def user_dir
116
+ # puts 'user_dir=' + @context.user_dir.to_s
117
+ @user_dir || "./"
118
+ end
119
+
120
+ def sw_set_data(data)
121
+ if data["attr_encoded"]
122
+ # new way, attributes are base 64 encoded
123
+ data = JSON.parse(Base64.decode64(data["attr_encoded"]))
124
+ end
125
+
126
+ data.each_pair do |k, v|
127
+ next unless k[0] == "@"
128
+ # puts "setting instance_variable #{k}=#{v}"
129
+ self.instance_variable_set(k, v)
130
+ end
131
+
132
+ end
133
+
134
+ def upload_if_needed(options={})
135
+ puts "No uploading in worker service."
136
+ end
137
+
138
+ alias_method :orig_queue, :queue
139
+ alias_method :orig_schedule, :schedule
140
+ alias_method :orig_status, :status
141
+
142
+ def queue(options={})
143
+ if IronWorker.queueing_enabled? && (!same_clazz? || options[:recursive])
144
+ orig_queue(options)
145
+ # data = sw_get_data()
146
+ # queue_other(self.class.name, data)
147
+ else
148
+ log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in queueing, pass in :recursive=>true to bypass this." : "Queuing disabled while loading.")
149
+ end
150
+ end
151
+
152
+ def schedule(schedule)
153
+ if IronWorker.queueing_enabled? && (!same_clazz? || schedule[:recursive])
154
+ orig_schedule(schedule)
155
+ # data = sw_get_data()
156
+ # schedule_other(self.class.name, data, schedule)
157
+ else
158
+ log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in scheduling." : "Scheduling disabled while loading.")
159
+ end
160
+
161
+ end
162
+
163
+ def status
164
+ if IronWorker.queueing_enabled?
165
+ orig_status
166
+ else
167
+ log "Status disabled while loading."
168
+ end
169
+ end
170
+
171
+ def same_clazz?
172
+ IronWorker.running_class == self.class
173
+ end
174
+
175
+ end
176
+
177
+
178
+ class Service < IronWorker::Api::Client
179
+ def upload(filename, class_name, options={})
180
+ #puts "Skipping upload, We don't upload from run.rb!"
181
+ # don't upload, should already be here.
182
+ end
183
+
184
+ def add_sw_params(hash_to_send)
185
+ hash_to_send["token"] = self.config.token
186
+ hash_to_send["project_id"] = self.config.project_id
187
+ hash_to_send["api_version"] = IronWorker.api_version
188
+ end
189
+ end
190
+
191
+
192
+ module UsedInWorker
193
+ def log(str)
194
+ puts str.to_s
195
+ end
196
+ end
197
+
198
+
199
+ end