iron_worker 2.4.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/README.md +352 -0
  4. data/iron_worker.gemspec +3 -8
  5. data/lib/iron_worker.rb +13 -51
  6. data/lib/iron_worker/api_client.rb +152 -0
  7. data/lib/iron_worker/version.rb +9 -1
  8. data/lib/iron_worker/worker_helper.rb +72 -0
  9. metadata +18 -183
  10. data/README.markdown +0 -73
  11. data/lib/generators/iron_worker/iron_worker_generator.rb +0 -13
  12. data/lib/generators/iron_worker/templates/template_worker.erb +0 -7
  13. data/lib/iron_worker/api.rb +0 -244
  14. data/lib/iron_worker/base.rb +0 -449
  15. data/lib/iron_worker/config.rb +0 -287
  16. data/lib/iron_worker/rails2_init.rb +0 -8
  17. data/lib/iron_worker/railtie.rb +0 -16
  18. data/lib/iron_worker/server/overrides.rb +0 -198
  19. data/lib/iron_worker/server/runner.rb +0 -82
  20. data/lib/iron_worker/service.rb +0 -644
  21. data/lib/iron_worker/used_in_worker.rb +0 -11
  22. data/lib/iron_worker/utils.rb +0 -11
  23. data/rails/init.rb +0 -0
  24. data/test/Gemfile +0 -36
  25. data/test/Gemfile.lock +0 -169
  26. data/test/README.md +0 -5
  27. data/test/batch_run.rb +0 -121
  28. data/test/config_sample.yml +0 -13
  29. data/test/models/cool_model.rb +0 -7
  30. data/test/models/db_model.rb +0 -6
  31. data/test/models/model_1.rb +0 -16
  32. data/test/models/model_2.rb +0 -16
  33. data/test/quick_run.rb +0 -36
  34. data/test/quick_schedule.rb +0 -18
  35. data/test/resources/something.yml +0 -4
  36. data/test/simple_script.rb +0 -11
  37. data/test/test_base.rb +0 -79
  38. data/test/test_database.rb +0 -31
  39. data/test/test_gems.rb +0 -31
  40. data/test/test_inheritance.rb +0 -25
  41. data/test/test_iron_worker.rb +0 -194
  42. data/test/test_merging.rb +0 -51
  43. data/test/test_non_worker_script.rb +0 -14
  44. data/test/test_rails.rb +0 -35
  45. data/test/test_scheduling.rb +0 -74
  46. data/test/test_scheduling_no_active_support.rb +0 -37
  47. data/test/test_service.rb +0 -28
  48. data/test/test_uploads.rb +0 -30
  49. data/test/test_webhooks.rb +0 -45
  50. data/test/v2quick.rb +0 -29
  51. data/test/workers/awesome_job.rb +0 -26
  52. data/test/workers/aws_s3_worker.rb +0 -10
  53. data/test/workers/big_data_worker.rb +0 -15
  54. data/test/workers/big_gems_worker.rb +0 -35
  55. data/test/workers/cool_worker.rb +0 -14
  56. data/test/workers/db_worker.rb +0 -27
  57. data/test/workers/fail_worker.rb +0 -10
  58. data/test/workers/gem_dependency_worker.rb +0 -20
  59. data/test/workers/hacker_worker.rb +0 -65
  60. data/test/workers/iw_test_worker.rb +0 -22
  61. data/test/workers/iw_test_worker_2.rb +0 -53
  62. data/test/workers/iw_test_worker_3.rb +0 -17
  63. data/test/workers/local_vs_remote_worker.rb +0 -9
  64. data/test/workers/merging_worker.rb +0 -40
  65. data/test/workers/mq_worker.rb +0 -21
  66. data/test/workers/one_line_worker.rb +0 -10
  67. data/test/workers/prawn_worker.rb +0 -32
  68. data/test/workers/progress_worker.rb +0 -26
  69. data/test/workers/qb_worker.rb +0 -17
  70. data/test/workers/rails_worker.rb +0 -24
  71. data/test/workers/rails_worker2.rb +0 -12
  72. data/test/workers/requiring_worker.rb +0 -4
  73. data/test/workers/running_back_worker.rb +0 -7
  74. data/test/workers/scheduled_worker.rb +0 -15
  75. data/test/workers/second_worker.rb +0 -16
  76. data/test/workers/third_worker.rb +0 -6
  77. data/test/workers/webhook_worker.rb +0 -7
  78. data/test/zip_vs_gzip.rb +0 -26
@@ -1,82 +0,0 @@
1
- # This is the file that gets executed on the server.
2
-
3
- def init_database_connection(sw_config)
4
- if sw_config
5
- db_config = sw_config['database']
6
- if db_config
7
- #@logger.info "Connecting to database using ActiveRecord..."
8
- require 'active_record'
9
- ActiveRecord::Base.establish_connection(db_config)
10
- end
11
- end
12
- end
13
-
14
- def init_mailer(sw_config)
15
- if sw_config
16
- mailer_config = sw_config['mailer']
17
- if mailer_config
18
- require 'action_mailer'
19
- ActionMailer::Base.raise_delivery_errors = true
20
- ActionMailer::Base.smtp_settings = mailer_config
21
- ActionMailer::Base.delivery_method = :smtp
22
- end
23
- end
24
- end
25
-
26
- def get_class_to_run(class_name)
27
- runner_class = constantize(class_name)
28
- return runner_class
29
- end
30
-
31
- # File activesupport/lib/active_support/inflector/methods.rb, line 107
32
- # Shoutout to the MIT License
33
- def constantize(camel_cased_word)
34
- names = camel_cased_word.split('::')
35
- names.shift if names.empty? || names.first.empty?
36
-
37
- constant = Object
38
- names.each do |name|
39
- constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
40
- end
41
- constant
42
- end
43
-
44
- def init_runner(runner_class, job_data, user_dir, task_id)
45
- # ensure initialize takes no arguments
46
- init_arity = runner_class.instance_method(:initialize).arity
47
- if init_arity == 0 || init_arity == -1
48
- # good. -1 can be if it's not defined at all
49
- else
50
- raise IronWorker::InvalidWorkerError, "Worker initialize method must accept zero arguments."
51
- end
52
- runner = runner_class.new
53
- runner.instance_variable_set(:@task_id, task_id)
54
- if job_data
55
- runner.instance_variable_set(:@job_data, job_data)
56
- runner.instance_variable_set(:@sw_config, job_data['sw_config'])
57
- runner.sw_set_data(job_data)
58
- end
59
- runner.instance_variable_set(:@user_dir, user_dir)
60
- runner
61
- end
62
-
63
- def init_worker_service_for_runner(job_data)
64
- IronWorker.configure do |config|
65
- sw_config = job_data['sw_config']
66
- config.token = sw_config['token']
67
- config.project_id = sw_config['project_id']
68
- config.scheme = sw_config['scheme'] if sw_config['scheme']
69
- config.host = sw_config['host'] if sw_config['host']
70
- config.port = sw_config['port'] if sw_config['port']
71
- db_config = sw_config['database']
72
- if db_config
73
- config.database = db_config
74
- end
75
- mailer_config = sw_config['mailer']
76
- if mailer_config && config.respond_to?(:mailer)
77
- config.mailer = mailer_config
78
- end
79
- config.global_attributes = sw_config['global_attributes'] if sw_config['global_attributes']
80
- end
81
- end
82
-
@@ -1,644 +0,0 @@
1
- require 'base64'
2
- require 'logger'
3
- begin
4
- # Little hack until everyone upgrades to rubyzip >= 1
5
- require 'zip'
6
- ZIPCLZ = Zip::File
7
- rescue Exception => ex
8
- require 'zip/zip'
9
- ZIPCLZ = Zip::ZipFile
10
- end
11
- require 'bundler'
12
- require 'rest_client'
13
- require 'json'
14
-
15
- require_relative 'api'
16
-
17
- module IronWorker
18
-
19
- class Service < IronWorker::Api::Client
20
-
21
- attr_accessor :config
22
-
23
- def initialize(token, options={})
24
- if options[:config]
25
- self.config = options[:config]
26
- else
27
- c = IronWorker::Config.new unless self.config
28
- c.token = token
29
- self.config = c
30
- end
31
- options[:version] = IronWorker.api_version
32
- options[:logger] = IronWorker.logger
33
- super("worker-aws-us-east-1.iron.io", token, options)
34
- self.host = self.config.host if self.config && self.config.host
35
- # automerge simple worker gem and dependencies
36
- self.config.merge_gem('rubyzip', :require => 'zip')
37
- self.config.merge_gem('rest-client')
38
- self.config.merge_gem('rest')
39
- self.config.merge_gem('iron_worker')
40
- IronWorker.logger.info 'IronWorker initialized.'
41
- IronWorker.logger.debug ' host = ' + self.host.inspect
42
- end
43
-
44
- # Options:
45
- # - :callback_url
46
- # - :merge => array of files to merge in with this file
47
- def upload(filename, class_name, options={})
48
- if self.config.no_upload
49
- IronWorker.logger.debug 'NO UPLOAD!'
50
- return
51
- end
52
- name = options[:name] || class_name
53
- project_id = get_project_id(options)
54
- tmp = Dir.tmpdir()
55
- md5file = "iron_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
56
- existing_md5 = nil
57
- md5_f = File.join(tmp, md5file)
58
- if File.exists?(md5_f)
59
- existing_md5 = IO.read(md5_f)
60
- end
61
-
62
-
63
- begin
64
-
65
- zip_filename = build_merged_file(filename, options[:merge], options[:unmerge], options[:merged_gems], options[:unmerged_gems], options[:merged_mailers], options[:merged_folders], class_name)
66
-
67
- # Check for code changes.
68
- zipfile = ZIPCLZ.open(zip_filename, ZIPCLZ::CREATE)
69
- crc = zipfile.entries.collect { |x| x.crc }.inject(:+)
70
- new_code = false
71
- if self.config.force_upload || crc.to_s != existing_md5
72
- IronWorker.logger.info "Uploading #{class_name}, code modified."
73
- File.open(md5_f, 'w') { |f| f.write(crc) }
74
- new_code = true
75
- # todo: delete md5 file if error occurs during upload process
76
- else
77
- # puts "#{class_name}: same code, not uploading"
78
- return
79
- end
80
-
81
- if new_code
82
- upload_code(name, zip_filename, 'runner.rb', :runtime => 'ruby')
83
- end
84
-
85
- rescue Exception => ex
86
- # if it errors, let's delete md5 since it wouldn't have uploaded.
87
- File.delete(md5_f) if File.exists?(md5_f)
88
- raise ex
89
- end
90
- end
91
-
92
- def get_server_gems
93
- hash = get("gems/list")
94
- JSON.parse(hash["gems"])
95
- end
96
-
97
- def logger
98
- IronWorker.logger
99
- end
100
-
101
-
102
- def self.heroku?
103
- !Gem::GemPathSearcher.private_instance_methods.include?(:_deprecated_initialize) rescue false
104
- end
105
-
106
- def webhook
107
-
108
- end
109
-
110
- def self.get_gem_path(gem_info)
111
- # gem_name =(gem_info[:require] || gem_info[:name].match(/^[a-zA-Z0-9\-_\.]+/)[0])
112
- gem_name =(gem_info[:name].match(/^[a-zA-Z0-9\-_\.]+/)[0])
113
- IronWorker.logger.debug "Searching for gem #{gem_name}..."
114
- gems= Gem::Specification.respond_to?(:each) ? Gem::Specification.find_all_by_name(gem_name) : Gem::GemPathSearcher.new.find_all(gem_name)
115
- if (Service.heroku?) && (!gems || gems.empty?)
116
- gems = Gem::GemPathSearcher.new.init_gemspecs.select { |gem| gem.name==gem_name }
117
- end
118
- IronWorker.logger.debug 'gems found=' + gems.inspect
119
- gems = gems.select { |g| g.version.version==gem_info[:version] } if gem_info[:version]
120
- if !gems.empty?
121
- gem = gems.last
122
- return gem, gem.full_gem_path
123
- else
124
- return nil, nil
125
- end
126
- end
127
-
128
- def gem_dependencies(list_of_gems)
129
- if Service.heroku?
130
- gem_specs = Gem::GemPathSearcher.new.init_gemspecs
131
- else
132
- gem_specs = Gem::Specification.all
133
- end
134
-
135
- IronWorker.logger.debug "Getting gem_dependencies.."
136
- deps = []
137
- dependendent_gems ={}
138
- list_of_gems.each_value do |v|
139
- IronWorker.logger.debug "Dependencies for #{v}"
140
- @deps = v[:gemspec].dependencies
141
- @deps.each do |d|
142
- IronWorker.logger.debug "dep: #{d}"
143
- deps << Bundler::DepProxy.new(d, 'ruby')
144
- end
145
- end
146
- filtered_deps = deps.select { |d| d.type != :development }
147
- IronWorker.logger.debug "filtered_deps=#{filtered_deps}"
148
- index = Bundler::Index.new
149
- gem_specs.collect { |s| index<<s }
150
- list = Bundler::Resolver.resolve(filtered_deps, index)
151
- list.each do |gemspec|
152
- next if list_of_gems.keys.include?(gemspec.name) || Config.system_gems.include?(gemspec.name)
153
- IronWorker.logger.debug "gemspec from bundler " + gemspec.inspect
154
- gi = IronWorker::MergeHelper.create_gem_info(gemspec.name, gemspec.version.version)
155
- dependendent_gems[gemspec.name] = gi
156
- end
157
- dependendent_gems
158
- end
159
-
160
- def build_merged_file(filename, merged, unmerge, merged_gems, unmerged_gems, merged_mailers, merged_folders, class_name)
161
-
162
- unmerged_gems ||= {}
163
-
164
- merge = IronWorker.config.merged.dup
165
- merge.merge!(merged) if merged
166
- if unmerge
167
- unmerge.each_pair do |x, y|
168
- deleted = merge.delete x
169
- IronWorker.logger.debug "Unmerging #{x}. Success? #{deleted}"
170
- end
171
- end
172
- merged = merge
173
- IronWorker.logger.debug 'merged=' + merged.inspect
174
- merged_gems = IronWorker.config.merged_gems.merge(merged_gems)
175
- IronWorker.logger.debug 'merged_gems=' + merged_gems.inspect
176
-
177
- if config.skip_auto_dependencies
178
- gems_dependencies = {}
179
- else
180
- gems_dependencies = gem_dependencies(merged_gems)
181
- IronWorker.logger.debug 'gem_dependencies=' + gems_dependencies.inspect
182
- # add dependencies to merged_gems
183
- #require gems dependencies
184
- gems_dependencies.each_pair do |k, gem|
185
- IronWorker.logger.debug "Bundling dependent gem #{gem[:name]}..."
186
- gem[:bypass_require] = true
187
- # merged_gems[k] ||= gem
188
- end
189
- end
190
- merged_gems = gems_dependencies.merge merged_gems
191
- # Now remove unmerged gems
192
- unmerged_gems = unmerged_gems.merge(IronWorker.config.unmerged_gems)
193
- unmerged_gems.each_pair do |k, v|
194
- IronWorker.logger.debug 'unmerging gem=' + k.inspect
195
- merged_gems.delete(k)
196
- end
197
- IronWorker.logger.debug 'merged_gems_after=' + merged_gems.inspect
198
-
199
- merged_mailers ||= {}
200
- merged_mailers = merged_mailers.merge(IronWorker.config.mailers) if IronWorker.config.mailers
201
-
202
- #tmp_file = File.join(Dir.tmpdir(), File.basename(filename))
203
- tmp_file = File.join(Dir.tmpdir(), 'runner.rb')
204
- File.open(tmp_file, "w") do |f|
205
-
206
- f.write("
207
- # Find environment (-e)
208
- dirname = ''
209
- i = 0
210
- task_data_file = nil
211
- task_id = nil
212
- #puts \"args for single file=\" + ARGV.inspect
213
- ARGV.each do |arg|
214
- if arg == \"-d\"
215
- # the user's writable directory
216
- dirname = ARGV[i+1]
217
- end
218
- if arg == \"-id\"
219
- # task_id
220
- task_id = ARGV[i+1]
221
- end
222
- if arg == \"-payload\"
223
- # path to job data
224
- task_data_file = ARGV[i+1]
225
- end
226
- i+=1
227
- end
228
- require 'json'
229
- ")
230
-
231
- # require merged gems
232
- merged_gems.each_pair do |k, gem|
233
- f.write "$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '/gems/#{gem[:name]}/lib'))\n"
234
- end
235
-
236
- merged_gems.each_pair do |k, gem|
237
- IronWorker.logger.debug "Bundling gem #{gem[:name]}..."
238
- #f.write "$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '/gems/#{gem[:name]}/lib'))\n"
239
- next if gem[:bypass_require]
240
- IronWorker.logger.debug 'writing requires: ' + gem[:require].inspect
241
- if gem[:require].nil?
242
- gem[:require] = []
243
- elsif gem[:require].is_a?(String)
244
- gem[:require] = [gem[:require]]
245
- end
246
- IronWorker.logger.debug "gem[:require]: " + gem[:require].inspect
247
- gem[:require].each do |r|
248
- IronWorker.logger.debug 'adding require to file ' + r.to_s
249
- f.write "require '#{r}'\n"
250
- end
251
- end
252
-
253
- File.open(File.join(File.dirname(__FILE__), 'server', 'overrides.rb'), 'r') do |fr|
254
- while line = fr.gets
255
- f.write line
256
- end
257
- end
258
-
259
- # Now we must disable queuing while loading up classes. This is from the overrides.rb file
260
- f.write("
261
- IronWorker.disable_queueing()
262
- ")
263
-
264
-
265
- File.open(File.join(File.dirname(__FILE__), "server", 'runner.rb'), 'r') do |fr|
266
- while line = fr.gets
267
- f.write line
268
- end
269
- end
270
-
271
- # load job data
272
- f.write("
273
- # Change to user directory
274
- #puts 'dirname=' + dirname.inspect
275
- Dir.chdir(dirname)
276
- # Load in job data
277
- payload = IO.read(task_data_file)
278
- IronWorker.payload = payload
279
- begin
280
- job_data = JSON.parse(payload)
281
- #puts 'payload=' + job_data.inspect
282
- sw_config = job_data['sw_config']
283
- IronWorker.task_data = job_data
284
-
285
- if IronWorker.task_data['rails']
286
- module ::Rails
287
- def self.version
288
- IronWorker.task_data['rails']['version']
289
- end
290
- def self.env
291
- IronWorker.task_data['rails']['env']
292
- end
293
- end
294
- end
295
- rescue JSON::ParserError => ex
296
- puts 'Payload is not json, raw payload can be accessed with IronWorker.payload.'
297
- end
298
- ")
299
-
300
- if IronWorker.config.extra_requires
301
- IronWorker.config.extra_requires.each do |r|
302
- f.write "require '#{r}'\n"
303
- end
304
- end
305
- if merged_mailers && !merged_mailers.empty?
306
- # todo: isn't 'action_mailer already required in railtie?
307
- f.write "require 'action_mailer'\n"
308
- f.write "init_mailer(sw_config)\n"
309
- f.write "ActionMailer::Base.prepend_view_path('templates')\n"
310
- end
311
- f.write "init_database_connection(sw_config)\n"
312
-
313
- merged.each_pair do |k, v|
314
- if v[:extname] == ".rb"
315
- f.write "require_relative '#{File.basename(v[:path])}'\n"
316
- end
317
- end
318
- merged_mailers.each_pair do |k, mailer|
319
- f.write "require_relative '#{mailer[:name]}'\n"
320
- end
321
- #end
322
- #f.write File.open(filename, 'r') { |mo| mo.read }
323
- f.write("require_relative '#{File.basename(filename)}'\n")
324
-
325
- f.write("
326
- clz_name = '#{class_name}' # job_data['class_name']
327
- runner_class = get_class_to_run(clz_name)
328
- IronWorker.running_class = runner_class
329
- runner = init_runner(runner_class, job_data, dirname, task_id)
330
- if IronWorker.task_data
331
- init_worker_service_for_runner(job_data)
332
- end
333
- # Now reenable after loading
334
- IronWorker.enable_queueing()
335
-
336
- # Let's run it!
337
- runner_return_data = runner.run
338
- ")
339
-
340
- end
341
- #puts 'funner.rb=' + tmp_file
342
- merge['runner.rb'] = {:path => tmp_file}
343
- #puts 'filename=' + filename
344
- merge[File.basename(filename)] = {:path => filename}
345
- #puts "merge before uniq! " + merge.inspect
346
- # puts "merge after uniq! " + merge.inspect
347
-
348
- fname2 = tmp_file + ".zip"
349
- # puts 'fname2=' + fname2
350
- # puts 'merged_file_array=' + merge.inspect
351
- #File.open(fname2, "w") do |f|
352
- File.delete(fname2) if File.exist?(fname2)
353
- #merging all gems and deps
354
- merged_gems.merge!(gems_dependencies)
355
- ZIPCLZ.open(fname2, 'w') do |f|
356
- if merged_gems && merged_gems.size > 0
357
- merged_gems.each_pair do |k, gem|
358
- next unless gem[:merge]
359
- # puts 'gem=' + gem.inspect
360
- path = gem[:path]
361
- if path
362
- #IronWorker.logger.debug "Collecting gem #{path}"
363
- paths_to_use = ["#{path}/*", "#{path}/lib/**/**"]
364
- if gem[:include_dirs]
365
- #IronWorker.logger.debug "including extra dirs: " + gem[:include_dirs].inspect
366
- gem[:include_dirs].each do |dir|
367
- paths_to_use << "#{path}/#{dir}/**/**"
368
- end
369
- end
370
- IronWorker.logger.debug 'paths_to_use: ' + paths_to_use.inspect
371
- Dir.glob(paths_to_use).each do |file|
372
- # todo: could check if directory and it not lib, skip it
373
- #IronWorker.logger.debug 'file for gem=' + file.inspect
374
- # puts 'gem2=' + gem.inspect
375
- zdest = "gems/#{gem[:name]}/#{file.sub(path+'/', '')}"
376
- # puts 'gem file=' + file.to_s
377
- #IronWorker.logger.debug 'zip dest=' + zdest
378
- f.add(zdest, file)
379
- end
380
- else
381
- if gem[:auto_merged]
382
- # todo: should only continue if the gem was auto merged.
383
- IronWorker.logger.warn "Gem #{gem[:name]} #{gem[:version]} was not found, continuing anyways."
384
- else
385
- raise "Gem #{gem[:name]} #{gem[:version]} was not found. This will occour when gem_name.gemspec is not the same as the gems primary require."
386
- end
387
-
388
- end
389
- end
390
- end
391
- if merged_folders && merged_folders.size > 0
392
- merged_folders.each do |folder, files|
393
- IronWorker.logger.debug "Collecting folder #{folder}"
394
- if files and files.size>0
395
- files.each do |file|
396
- zdest = "#{Digest::MD5.hexdigest(folder)}/#{file.sub(':', '_').sub('/', '_')}"
397
- #IronWorker.logger.debug 'put file to=' + zdest
398
- f.add(zdest, file)
399
- end
400
- end
401
- end
402
- end
403
-
404
- IronWorker.logger.debug "merge=" + merge.inspect
405
- merge.each_pair do |k, v|
406
- IronWorker.logger.debug "merging k=#{k.inspect} v=#{v.inspect} into #{filename}"
407
- f.add(File.basename(v[:path]), v[:path])
408
- end
409
- if merged_mailers && merged_mailers.size > 0
410
- # puts " MERGED MAILERS" + merged_mailers.inspect
411
- merged_mailers.each_pair do |k, mailer|
412
- IronWorker.logger.debug "Collecting mailer #{mailer[:name]}"
413
- f.add(File.basename(mailer[:filename]), mailer[:filename])
414
- path = mailer[:path_to_templates]
415
- Dir["#{path}/**/**"].each do |file|
416
- zdest = "templates/#{mailer[:name]}/#{file.sub(path+'/', '')}"
417
- f.add(zdest, file)
418
- end
419
- path = mailer[:path_to_layouts]
420
- Dir["#{path}/**/**"].each do |file|
421
- zdest = "templates/layouts/#{file.sub(path+'/', '')}"
422
- f.add(zdest, file)
423
- end
424
- end
425
- end
426
- end
427
- fname2
428
- end
429
-
430
- # This will package up files into a zip file ready for uploading.
431
- def package_code(files)
432
- fname2 = "package.zip"
433
- File.delete(fname2) if File.exist?(fname2)
434
- ZIPCLZ.open(fname2, 'w') do |f|
435
- files.each do |file|
436
- f.add(file, file)
437
- end
438
- end
439
- fname2
440
- end
441
-
442
- # options:
443
- # :runtime => 'ruby', 'python', 'node', 'java', 'go'
444
- def upload_code(name, package_file, exec_file, options={})
445
- IronWorker.logger.info 'file size to upload: ' + File.size(package_file).to_s
446
- options = {
447
- "name" => name,
448
- "standalone" => true,
449
- "runtime" => options[:runtime] || "ruby",
450
- "file_name" => exec_file # File.basename(filename)
451
- }
452
- #puts 'options for upload=' + options.inspect
453
- IronWorker.logger.info "Uploading now..."
454
- ret = post_file("#{project_url_prefix(get_project_id(options))}codes", File.new(package_file), options)
455
- IronWorker.logger.info "Done uploading."
456
- return ret
457
- end
458
-
459
- def project_url_prefix(project_id = 0)
460
- # IronWorker.logger.info "project_url_prefix, project_id = " + project_id.inspect
461
- if project_id == 0
462
- return false
463
- project_id = config.project_id
464
- end
465
- "projects/#{project_id}/"
466
- end
467
-
468
- def wait_until_complete(task_id, options={})
469
- tries = 0
470
- status = nil
471
- sleep 1
472
- while tries < 100
473
- status = options[:schedule] ? schedule_status(task_id) : status(task_id)
474
- puts "Waiting... status=" + status["status"]
475
- if options[:schedule]
476
- if status["status"] == "complete"
477
- break
478
- end
479
- else
480
- if status["status"] != "queued" && status["status"] != "running"
481
- break
482
- end
483
- end
484
- sleep 2
485
- end
486
- status
487
- end
488
-
489
- def add_sw_params(hash_to_send)
490
- # todo: remove secret key?? Can use worker service from within a worker without it now
491
- hash_to_send["oauth"] = self.token
492
- hash_to_send["api_version"] = IronWorker.api_version
493
- end
494
-
495
- def check_config
496
- if self.config.nil? || self.config.token.nil? || self.config.project_id.nil?
497
- raise "Invalid IronWorker configuration, token and project_id required."
498
- end
499
- end
500
-
501
- def enqueue(name, data={}, options={})
502
- queue(name, data, options)
503
- end
504
-
505
- # name: The name of previously upload worker code, eg: MySuperWorker
506
- # data: Arbitrary hash of your own data that your task will need to run.
507
- def queue(name, data={}, options={})
508
- puts "Queuing #{name}..."
509
- check_config
510
- unless data.is_a?(Array)
511
- data = [data]
512
- end
513
- # Now we need to add class_name to the payload
514
- tasks = []
515
- data.each do |d|
516
- d['class_name'] = name unless d['class_name']
517
- task = {}
518
- task["payload"] = d.to_json
519
- task["code_name"] = name
520
- task.merge!(options)
521
- tasks << task
522
- end
523
- name = options[:name] || name
524
- hash_to_send = {}
525
- hash_to_send["options"] = options
526
- hash_to_send["tasks"] = tasks
527
- add_sw_params(hash_to_send)
528
- if defined?(RAILS_ENV)
529
- # todo: REMOVE THIS
530
- hash_to_send["rails_env"] = RAILS_ENV
531
- end
532
- queue_raw(name, hash_to_send, options)
533
- end
534
-
535
- def queue_raw(name, data={}, options={})
536
- params = nil
537
- hash_to_send = data
538
- #hash_to_send["class_name"] = name unless hash_to_send["class_name"]
539
- hash_to_send["name"] = name unless hash_to_send["name"]
540
- uri = project_url_prefix(get_project_id(options)) + "tasks"
541
- IronWorker.logger.debug 'queue_raw , uri = ' + uri
542
- ret = post(uri, hash_to_send)
543
- ret
544
- end
545
-
546
- #
547
- # schedule: hash of scheduling options that can include:
548
- # Required:
549
- # - start_at: Time of first run - DateTime or Time object.
550
- # Optional:
551
- # - run_every: Time in seconds between runs. If ommitted, task will only run once.
552
- # - delay_type: Fixed Rate or Fixed Delay. Default is fixed_delay.
553
- # - end_at: Scheduled task will stop running after this date (optional, if ommitted, runs forever or until cancelled)
554
- # - run_times: Task will run exactly :run_times. For instance if :run_times is 5, then the task will run 5 times.
555
- #
556
- def schedule(name, data, schedule)
557
- puts "Scheduling #{name}..."
558
- raise "Schedule must be a hash." if !schedule.is_a? Hash
559
- hash_to_send = {}
560
- schedules = []
561
- schedule["payload"] = data.to_json
562
- schedule["name"] = name unless schedule["name"]
563
- schedule["code_name"] = name unless schedule["code_name"]
564
- schedules << schedule
565
- hash_to_send["schedules"] = schedules
566
- add_sw_params(hash_to_send)
567
- # puts ' about to send ' + hash_to_send.inspect
568
- uri = project_url_prefix(get_project_id(data)) + "schedules"
569
- ret = post(uri, hash_to_send)
570
- ret
571
- end
572
-
573
- def cancel_schedule(scheduled_task_id, options={})
574
- raise "Must include a schedule id." if scheduled_task_id.blank?
575
- hash_to_send = {}
576
- hash_to_send["schedule_id"] = scheduled_task_id
577
- uri = "#{project_url_prefix(get_project_id(options))}schedules/#{scheduled_task_id}/cancel"
578
- post(uri, hash_to_send)
579
- end
580
-
581
- def get_project_id(options={})
582
- options[:project_id] || config.project_id
583
- end
584
-
585
- def codes(options={})
586
- hash_to_send = options
587
- uri = "projects/" + get_project_id(options) + "/codes"
588
- ret = get(uri, hash_to_send)
589
- ret
590
- end
591
-
592
- def schedules(options={})
593
- hash_to_send = options
594
- uri = "projects/" + get_project_id(options) + "/schedules"
595
- ret = get(uri, hash_to_send)
596
- ret
597
- end
598
-
599
- def jobs(options={})
600
- puts 'Service.jobs is deprecated, use Service.tasks instead'
601
- tasks(options)
602
- end
603
-
604
- def tasks(options={})
605
- hash_to_send = options
606
- uri = "projects/" + get_project_id(options) + "/tasks"
607
- ret = get(uri, hash_to_send)
608
- ret
609
- end
610
-
611
- def get_log(job_id, options={})
612
- puts 'Service.get_log is deprecated, use Service.log instead'
613
- log(job_id, options)
614
- end
615
-
616
- def log(task_id, options={})
617
- data = options
618
- ret = get("#{project_url_prefix(get_project_id(options))}tasks/#{task_id}/log", data, :parse => false)
619
- ret
620
- end
621
-
622
-
623
- def status(task_id, options={})
624
- data = {"task_id" => task_id}
625
- ret = get("#{project_url_prefix(get_project_id(options))}tasks/#{task_id}", data)
626
- ret
627
- end
628
-
629
- def schedule_status(schedule_id, options={})
630
- data = {"schedule_id" => schedule_id}
631
- ret = get("#{project_url_prefix(get_project_id(options))}schedules/#{schedule_id}", data)
632
- ret
633
- end
634
-
635
- # data is a hash, should include 'percent' and 'msg'
636
- def set_progress(task_id, options={})
637
- #data={"data"=>data, "task_id"=>task_id}
638
- post("#{project_url_prefix(get_project_id(options))}tasks/#{task_id}/progress", options)
639
- end
640
-
641
-
642
- end
643
-
644
- end