iron_worker 2.4.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/README.md +352 -0
  4. data/iron_worker.gemspec +3 -8
  5. data/lib/iron_worker.rb +13 -51
  6. data/lib/iron_worker/api_client.rb +152 -0
  7. data/lib/iron_worker/version.rb +9 -1
  8. data/lib/iron_worker/worker_helper.rb +72 -0
  9. metadata +18 -183
  10. data/README.markdown +0 -73
  11. data/lib/generators/iron_worker/iron_worker_generator.rb +0 -13
  12. data/lib/generators/iron_worker/templates/template_worker.erb +0 -7
  13. data/lib/iron_worker/api.rb +0 -244
  14. data/lib/iron_worker/base.rb +0 -449
  15. data/lib/iron_worker/config.rb +0 -287
  16. data/lib/iron_worker/rails2_init.rb +0 -8
  17. data/lib/iron_worker/railtie.rb +0 -16
  18. data/lib/iron_worker/server/overrides.rb +0 -198
  19. data/lib/iron_worker/server/runner.rb +0 -82
  20. data/lib/iron_worker/service.rb +0 -644
  21. data/lib/iron_worker/used_in_worker.rb +0 -11
  22. data/lib/iron_worker/utils.rb +0 -11
  23. data/rails/init.rb +0 -0
  24. data/test/Gemfile +0 -36
  25. data/test/Gemfile.lock +0 -169
  26. data/test/README.md +0 -5
  27. data/test/batch_run.rb +0 -121
  28. data/test/config_sample.yml +0 -13
  29. data/test/models/cool_model.rb +0 -7
  30. data/test/models/db_model.rb +0 -6
  31. data/test/models/model_1.rb +0 -16
  32. data/test/models/model_2.rb +0 -16
  33. data/test/quick_run.rb +0 -36
  34. data/test/quick_schedule.rb +0 -18
  35. data/test/resources/something.yml +0 -4
  36. data/test/simple_script.rb +0 -11
  37. data/test/test_base.rb +0 -79
  38. data/test/test_database.rb +0 -31
  39. data/test/test_gems.rb +0 -31
  40. data/test/test_inheritance.rb +0 -25
  41. data/test/test_iron_worker.rb +0 -194
  42. data/test/test_merging.rb +0 -51
  43. data/test/test_non_worker_script.rb +0 -14
  44. data/test/test_rails.rb +0 -35
  45. data/test/test_scheduling.rb +0 -74
  46. data/test/test_scheduling_no_active_support.rb +0 -37
  47. data/test/test_service.rb +0 -28
  48. data/test/test_uploads.rb +0 -30
  49. data/test/test_webhooks.rb +0 -45
  50. data/test/v2quick.rb +0 -29
  51. data/test/workers/awesome_job.rb +0 -26
  52. data/test/workers/aws_s3_worker.rb +0 -10
  53. data/test/workers/big_data_worker.rb +0 -15
  54. data/test/workers/big_gems_worker.rb +0 -35
  55. data/test/workers/cool_worker.rb +0 -14
  56. data/test/workers/db_worker.rb +0 -27
  57. data/test/workers/fail_worker.rb +0 -10
  58. data/test/workers/gem_dependency_worker.rb +0 -20
  59. data/test/workers/hacker_worker.rb +0 -65
  60. data/test/workers/iw_test_worker.rb +0 -22
  61. data/test/workers/iw_test_worker_2.rb +0 -53
  62. data/test/workers/iw_test_worker_3.rb +0 -17
  63. data/test/workers/local_vs_remote_worker.rb +0 -9
  64. data/test/workers/merging_worker.rb +0 -40
  65. data/test/workers/mq_worker.rb +0 -21
  66. data/test/workers/one_line_worker.rb +0 -10
  67. data/test/workers/prawn_worker.rb +0 -32
  68. data/test/workers/progress_worker.rb +0 -26
  69. data/test/workers/qb_worker.rb +0 -17
  70. data/test/workers/rails_worker.rb +0 -24
  71. data/test/workers/rails_worker2.rb +0 -12
  72. data/test/workers/requiring_worker.rb +0 -4
  73. data/test/workers/running_back_worker.rb +0 -7
  74. data/test/workers/scheduled_worker.rb +0 -15
  75. data/test/workers/second_worker.rb +0 -16
  76. data/test/workers/third_worker.rb +0 -6
  77. data/test/workers/webhook_worker.rb +0 -7
  78. data/test/zip_vs_gzip.rb +0 -26
@@ -1,287 +0,0 @@
1
- module IronWorker
2
-
3
-
4
- # Config is used to setup the IronWorker client.
5
- # You must set the access_key and secret_key.
6
- #
7
- # config.global_attributes allows you to specify attributes that will automatically be set on every worker,
8
- # this is good for database connection information or things that will be used across the board.
9
- #
10
- # config.database configures a database connection. If specified like ActiveRecord, IronWorker will automatically establish a connection
11
- # for you before running your worker.
12
- class Config
13
- attr_accessor :token,
14
- :project_id,
15
- :scheme,
16
- :host,
17
- :port,
18
- :global_attributes,
19
- :models,
20
- :mailers,
21
- #:gems, # todo: move anything that uses this to merged_gems
22
- :database,
23
- :mailer,
24
- :extra_requires,
25
- #:auto_merge,
26
- :server_gems,
27
- :merged,
28
- :unmerged,
29
- :merged_gems,
30
- :unmerged_gems,
31
- :force_upload, :no_upload,
32
- :beta, # for setting beta feature flags
33
- :skip_auto_dependencies
34
-
35
- def self.system_gems
36
- ['yajl-ruby', 'bson_ext', 'curb', 'em-http-request', 'eventmachine', 'mysql2', 'net-scp', 'net-sftp', 'net-ssh', 'nokogiri', 'rmagick', 'sqlite3', 'typhoeus']
37
- end
38
-
39
- def initialize
40
- @global_attributes = {}
41
- @extra_requires = []
42
- @merged = {}
43
- @unmerged = {}
44
- @merged_gems = {}
45
- @unmerged_gems = {}
46
- @mailers = {}
47
- @beta = {}
48
-
49
- end
50
-
51
- def access_key=(x)
52
- raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
53
- end
54
-
55
- def secret_key=(x)
56
- raise "IronWorker Config Error: access_key and secret_key are no longer used. The new IronWorker gem requires a couple of small configuration changes, please see: http://docs.IronWorker.com/ruby/new-gem-v2-update-guide for information."
57
- end
58
-
59
- #@gems_to_skip = ['actionmailer', 'actionpack', 'activemodel', 'activeresource', 'activesupport',
60
- # 'bundler',
61
- # 'mail',
62
- # 'mysql2',
63
- # 'rails',
64
- # 'tzinfo' # HUGE!
65
- #]
66
-
67
- #def self.gems_to_skip
68
- # @gems_to_skip
69
- #end
70
-
71
- def bundle=(activate)
72
- if activate
73
- IronWorker.logger.info "Initializing IronWorker for Bundler..."
74
- IronWorker.configure do |c2|
75
- c2.merged_gems.merge!(get_required_gems)
76
- IronWorker.logger.debug "List of gems from bundler:#{c2.merged_gems.inspect}"
77
- end
78
- end
79
- end
80
-
81
- def auto_merge=(b)
82
- if b
83
- IronWorker.logger.info "Initializing IronWorker for Rails 3..."
84
- start_time = Time.now
85
- IronWorker.configure do |c2|
86
- models_path = File.join(Rails.root, 'app/models/*.rb')
87
- models = Dir.glob(models_path)
88
- c2.models = models
89
- models.each { |model| c2.merge(model) }
90
- mailers_path = File.join(Rails.root, 'app/mailers/*.rb')
91
- Dir.glob(mailers_path).collect { |m| c2.mailers[File.basename(m)] = {:filename=>m, :name => File.basename(m), :path_to_layouts => File.join(Rails.root, "app/views/layouts"), :path_to_templates=>File.join(Rails.root, "app/views/#{File.basename(m, File.extname(m))}")} }
92
- c2.extra_requires += ['active_support/core_ext', 'action_mailer']
93
- #puts 'DB FILE=' + File.join(Rails.root, 'config', 'database.yml').to_s
94
- if defined?(ActiveRecord) && File.exist?(File.join(Rails.root, 'config', 'database.yml'))
95
- c2.extra_requires += ['active_record']
96
- c2.database = Rails.configuration.database_configuration[Rails.env]
97
- else
98
- #puts 'NOT DOING ACTIVERECORD'
99
- end
100
-
101
- if defined?(ActionMailer) && ActionMailer::Base.smtp_settings
102
- c2.mailer = ActionMailer::Base.smtp_settings
103
- end
104
- c2.merged_gems.merge!(get_required_gems)
105
- IronWorker.logger.debug "MODELS " + c2.models.inspect
106
- IronWorker.logger.debug "MAILERS " + c2.mailers.inspect
107
- IronWorker.logger.debug "DATABASE " + c2.database.inspect
108
- #IronWorker.logger.debug "GEMS " + c2.gems.inspect
109
- end
110
- end_time = Time.now
111
- IronWorker.logger.info "IronWorker initialized. Duration: #{((end_time.to_f-start_time.to_f) * 1000.0).to_i} ms"
112
- end
113
- end
114
-
115
-
116
- def get_required_gems
117
- #skipping if bundler not defined or not initialized
118
- return {} unless defined?(Bundler) && Bundler.instance_variables.include?(:@setup)
119
- gems_in_gemfile = Bundler.environment.dependencies.select { |d| d.groups.include?(:default) }
120
- IronWorker.logger.debug 'gems in gemfile=' + gems_in_gemfile.inspect
121
- gems = {}
122
- specs = Bundler.load.specs
123
- IronWorker.logger.debug 'Bundler specs=' + specs.inspect
124
- #IronWorker.logger.debug "gems_to_skip=" + self.class.gems_to_skip.inspect
125
- specs.each do |spec|
126
- IronWorker.logger.debug 'spec.name=' + spec.name.inspect
127
- IronWorker.logger.debug 'spec=' + spec.inspect
128
- #if self.class.gems_to_skip.include?(spec.name)
129
- # IronWorker.logger.debug "Skipping #{spec.name}"
130
- # next
131
- #end
132
- # next if dep.name=='rails' #monkey patch
133
- gem_info = {:name=>spec.name, :version=>spec.version}
134
- gem_info[:auto_merged] = true
135
- gem_info[:merge] = spec.extensions.length == 0 #merging only non binary gems
136
- gem_info[:bypass_require] = true #don't require gem'
137
- # Now find dependency in gemfile in case user set the require
138
- dep = gems_in_gemfile.find { |g| g.name == gem_info[:name] }
139
- if dep
140
- IronWorker.logger.debug 'dep found in gemfile: ' + dep.inspect
141
- IronWorker.logger.debug 'autorequire=' + dep.autorequire.inspect
142
- gem_info[:require] = dep.autorequire if dep.autorequire
143
- # spec = specs.find { |g| g.name==gem_info[:name] }
144
- end
145
- gem_info[:version] = spec.version.to_s
146
- gems[gem_info[:name]] = gem_info
147
- gemspec, path = IronWorker::Service.get_gem_path(gem_info)
148
- if path
149
- gem_info[:gemspec] = gemspec
150
- gem_info[:path] = path
151
- if gem_info[:require].nil? && dep
152
- # see if we should try to require this in our worker
153
- require_path = gem_info[:path] + "/lib/#{gem_info[:name]}.rb"
154
- IronWorker.logger.debug "require_path=" + require_path
155
- if File.exists?(require_path)
156
- IronWorker.logger.debug "File exists for require"
157
- gem_info[:require] = gem_info[:name]
158
- else
159
- IronWorker.logger.debug "no require"
160
- # gem_info[:no_require] = true
161
- end
162
- end
163
- else
164
- IronWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
165
- end
166
- # else
167
- # IronWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
168
- # raise "Could not find gem spec for #{gem_info[:name]}"
169
- # end
170
- end
171
- gems
172
- end
173
-
174
- def get_server_gems
175
- return []
176
- # skipping this now, don't want any server dependencies if possible
177
- self.server_gems = IronWorker.service.get_server_gems unless self.server_gems
178
- self.server_gems
179
- end
180
-
181
- def get_atts_to_send
182
- config_data = {}
183
- config_data['token'] = token
184
- config_data['project_id'] = project_id
185
- config_data['database'] = self.database if self.database
186
- config_data['mailer'] = self.mailer if self.mailer
187
- config_data['global_attributes'] = self.global_attributes if self.global_attributes
188
- config_data['scheme'] = self.scheme if self.scheme
189
- config_data['host'] = self.host if self.host
190
- config_data['port'] = self.port if self.port
191
- config_data
192
- end
193
-
194
- def merge(file)
195
- f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
196
- fbase = f2[:basename]
197
- ret = f2
198
- @merged[fbase] = ret
199
- ret
200
- end
201
-
202
- def unmerge(file)
203
- f2 = IronWorker::MergeHelper.check_for_file(file, caller[2])
204
- fbase = f2[:basename]
205
- @unmerged[fbase] =f2
206
- @merged.delete(fbase)
207
- end
208
-
209
- # Merge a gem globally here
210
- def merge_gem(gem_name, options={})
211
- merged_gems[gem_name.to_s] = IronWorker::MergeHelper.create_gem_info(gem_name, options)
212
- end
213
-
214
- # Unmerge a global gem
215
- def unmerge_gem(gem_name)
216
- gs = gem_name.to_s
217
- gem_info = {:name=>gs}
218
- unmerged_gems[gs] = gem_info
219
- merged_gems.delete(gs)
220
- end
221
-
222
- end
223
-
224
-
225
- class MergeHelper
226
-
227
- # callerr is original file that is calling the merge function, ie: your worker.
228
- # See Base for examples.
229
- def self.check_for_file(f, callerr)
230
- IronWorker.logger.debug 'Checking for ' + f.to_s
231
- f = f.to_str
232
- f_ext = File.extname(f)
233
- if f_ext.empty?
234
- f_ext = ".rb"
235
- f << f_ext
236
- end
237
- exists = false
238
- if File.exist? f
239
- exists = true
240
- else
241
- # try relative
242
- # p caller
243
- f2 = File.join(File.dirname(callerr), f)
244
- #puts 'f2=' + f2
245
- if File.exist? f2
246
- exists = true
247
- f = f2
248
- end
249
- end
250
- unless exists
251
- raise "File not found: " + f
252
- end
253
- f = File.expand_path(f)
254
- require f if f_ext == '.rb'
255
- ret = {}
256
- ret[:path] = f
257
- ret[:extname] = f_ext
258
- ret[:basename] = File.basename(f)
259
- ret[:name] = ret[:basename]
260
- ret
261
- end
262
-
263
- def self.create_gem_info(gem_name, options={})
264
- gem_info = {:name=>gem_name, :merge=>true}
265
- if options.is_a?(Hash)
266
- gem_info.merge!(options)
267
- if options[:include_dirs]
268
- gem_info[:include_dirs] = options[:include_dirs].is_a?(Array) ? options[:include_dirs] : [options[:include_dirs]]
269
- end
270
- else
271
- gem_info[:version] = options
272
- end
273
-
274
- gemspec, path = IronWorker::Service.get_gem_path(gem_info)
275
- IronWorker.logger.debug "Gem path=#{path}"
276
- if !path
277
- raise "Gem '#{gem_name}' not found."
278
- end
279
- gem_info[:gemspec] = gemspec
280
- gem_info[:path] = path
281
- gem_info[:require] ||= gem_name
282
- gem_info
283
- end
284
- end
285
-
286
- end
287
-
@@ -1,8 +0,0 @@
1
- #puts "Initializing list of Rails models..."
2
- IronWorker.configure do |config|
3
- path = File.join(Rails.root, 'app/models/*.rb')
4
- # puts 'path=' + path
5
- config.models = Dir.glob(path)
6
- config.extra_requires += ['active_support/core_ext', 'active_record', 'action_mailer']
7
- # puts 'config.models=' + config.models.inspect
8
- end
@@ -1,16 +0,0 @@
1
- # see http://api.rubyonrails.org/classes/Rails/Railtie.html
2
-
3
- require 'iron_worker'
4
- require 'rails'
5
-
6
- module IronWorker
7
- class Railtie < Rails::Railtie
8
-
9
-
10
- initializer "iron_worker.configure_rails_initialization" do |app|
11
-
12
- end
13
-
14
-
15
- end
16
- end
@@ -1,198 +0,0 @@
1
- # This is used when a bad worker is uploaded.
2
-
3
- module IronWorker
4
-
5
- class InvalidWorkerError < StandardError;
6
- end
7
-
8
- class << self
9
- def running_class=(rc)
10
- @running_class = rc
11
- end
12
-
13
- def running_class
14
- @running_class
15
- end
16
-
17
- def task_data=(td)
18
- @task_data = td
19
- end
20
-
21
- def task_data
22
- @task_data
23
- end
24
-
25
- def payload=(td)
26
- @payload = td
27
- end
28
-
29
- def payload
30
- @payload
31
- end
32
-
33
- def is_local?
34
- !is_remote?
35
- end
36
-
37
- def is_remote?
38
- true
39
- end
40
- end
41
-
42
- def self.disable_queueing()
43
- @queueing_enabled = false
44
- end
45
-
46
- def self.enable_queueing()
47
- @queueing_enabled = true
48
- end
49
-
50
- def self.queueing_enabled?
51
- @queueing_enabled
52
- end
53
-
54
- class Config
55
-
56
- def merge(file)
57
- end
58
-
59
- def unmerge(file)
60
- end
61
-
62
- def merge_gem(gem_name, options={})
63
- end
64
-
65
- def unmerge_gem(gem_name)
66
- end
67
-
68
- def merge_folder(path)
69
- end
70
-
71
-
72
- end
73
-
74
-
75
- class Base
76
-
77
- class << self
78
-
79
- def merge(*files)
80
- end
81
-
82
- def merge_folder(path)
83
- end
84
-
85
- def unmerge(*files)
86
- # ignore this here
87
- end
88
-
89
- def merge_mailer(mailer, params=nil)
90
- end
91
-
92
- def merge_gem(gem, version=nil)
93
- end
94
-
95
- def merge_worker(file, class_name)
96
-
97
- end
98
-
99
- end
100
-
101
- def log(str)
102
- puts str.to_s
103
- end
104
-
105
- def set_progress(hash)
106
- #puts 'set_progress self=' + self.inspect
107
- IronWorker.service.set_progress(self.task_id, hash)
108
- end
109
-
110
- def something
111
- puts 'which class? ' + self.class.name
112
- end
113
-
114
- def user_dir
115
- # puts 'user_dir=' + @context.user_dir.to_s
116
- @user_dir || "./"
117
- end
118
-
119
- def sw_set_data(data)
120
- if data["attr_encoded"]
121
- # new way, attributes are base 64 encoded
122
- data = JSON.parse(Base64.decode64(data["attr_encoded"]))
123
- end
124
-
125
- data.each_pair do |k, v|
126
- next unless k[0] == "@"
127
- # puts "setting instance_variable #{k}=#{v}"
128
- self.instance_variable_set(k, v)
129
- end
130
-
131
- end
132
-
133
- def upload_if_needed(options={})
134
- puts "No uploading in worker service."
135
- end
136
-
137
- alias_method :orig_queue, :queue
138
- alias_method :orig_schedule, :schedule
139
- alias_method :orig_status, :status
140
-
141
- def queue(options={})
142
- if IronWorker.queueing_enabled? && (!same_clazz? || options[:recursive])
143
- orig_queue(options)
144
- # data = sw_get_data()
145
- # queue_other(self.class.name, data)
146
- else
147
- log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in queueing, pass in :recursive=>true to bypass this." : "Queuing disabled while loading.")
148
- end
149
- end
150
-
151
- def schedule(schedule)
152
- if IronWorker.queueing_enabled? && (!same_clazz? || schedule[:recursive])
153
- orig_schedule(schedule)
154
- # data = sw_get_data()
155
- # schedule_other(self.class.name, data, schedule)
156
- else
157
- log (IronWorker.queueing_enabled? ? "WARNING: Recursion detected in scheduling." : "Scheduling disabled while loading.")
158
- end
159
-
160
- end
161
-
162
- def status
163
- if IronWorker.queueing_enabled?
164
- orig_status
165
- else
166
- log "Status disabled while loading."
167
- end
168
- end
169
-
170
- def same_clazz?
171
- IronWorker.running_class == self.class
172
- end
173
-
174
- end
175
-
176
-
177
- class Service < IronWorker::Api::Client
178
- def upload(filename, class_name, options={})
179
- #puts "Skipping upload, We don't upload from run.rb!"
180
- # don't upload, should already be here.
181
- end
182
-
183
- def add_sw_params(hash_to_send)
184
- hash_to_send["token"] = self.config.token
185
- hash_to_send["project_id"] = self.config.project_id
186
- hash_to_send["api_version"] = IronWorker.api_version
187
- end
188
- end
189
-
190
-
191
- module UsedInWorker
192
- def log(str)
193
- puts str.to_s
194
- end
195
- end
196
-
197
-
198
- end