simple_worker 2.0.0.beta.3 → 2.0.0.beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.markdown CHANGED
@@ -13,11 +13,11 @@ Install SimpleWorker Gem
13
13
  Configure SimpleWorker
14
14
  ----------------------
15
15
 
16
- You really just need your access keys.
16
+ You really just need your token, which you can get [here][2]
17
+ [2]: http://simpleworker.com/tokens
17
18
 
18
19
  SimpleWorker.configure do |config|
19
- config.access_key = ACCESS_KEY
20
- config.secret_key = SECRET_KEY
20
+ config.token = TOKEN
21
21
  end
22
22
 
23
23
  Generate a Worker with Rails
@@ -30,24 +30,28 @@ module SimpleWorker
30
30
  # host: endpoint url for service
31
31
  class Client
32
32
 
33
- @@logger = Logger.new(STDOUT)
34
- @@logger.level = Logger::INFO
33
+ attr_accessor :host, :port, :token, :version, :config
35
34
 
36
- def self.logger
37
- @@logger
38
- end
35
+ def initialize(host, token, options={})
36
+ @host = host
37
+ @config = options[:config]
38
+ @port = options[:port] || @config.port || 80
39
+ @token = token
40
+ @version = options[:version]
41
+ @logger = options[:logger]
39
42
 
40
- attr_accessor :host, :access_key, :secret_key, :version
43
+ end
41
44
 
42
- def initialize(host, access_key, secret_key, options={})
43
- @host = host
44
- @access_key = access_key
45
- @secret_key = secret_key
45
+ def url(command_path)
46
+ url = "http://#{host}:#{port}/#{@version}/#{command_path}"
47
+ # @logger.debug "url: " + url.to_s
48
+ url
46
49
  end
47
50
 
48
51
  def process_ex(ex)
49
52
  body = ex.http_body
50
- @@logger.debug 'EX BODY=' + body.to_s
53
+ @logger.debug 'EX http_code: ' + ex.http_code.to_s
54
+ @logger.debug 'EX BODY=' + body.to_s
51
55
  decoded_ex = JSON.parse(body)
52
56
  exception = Exception.new(ex.message + ": " + decoded_ex["msg"])
53
57
  exception.set_backtrace(decoded_ex["backtrace"].split(",")) if decoded_ex["backtrace"]
@@ -55,25 +59,52 @@ module SimpleWorker
55
59
  end
56
60
 
57
61
  def get(method, params={}, options={})
58
- begin
62
+ #begin
59
63
  # ClientHelper.run_http(host, access_key, secret_key, :get, method, nil, params)
60
- parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
61
- rescue RestClient::Exception => ex
62
- process_ex(ex)
63
- end
64
+ full_url = url(method)
65
+ all_params = add_params(method,params)
66
+
67
+ url_plus_params = append_params(full_url, all_params)
68
+ resp = RestClient.get(url_plus_params, headers)
69
+
70
+ parse_response(resp, options)
71
+
72
+ # Was:
73
+ #parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
74
+ #rescue RestClient::Exception => ex
75
+ # process_ex(ex)
76
+ #end
64
77
  end
65
78
 
66
79
  def post_file(method, file, params={}, options={})
67
80
  begin
68
- parse_response RestClient.post(url(method), add_params(method, params).merge!({:file=>file}), :multipart => true), options
81
+ #params.delete("runtime")
82
+ #params["runtime"]='ruby'
83
+ #params.delete("file_name")
84
+ #params["file_name"] = "runner.rb"
85
+ data = add_params(method, params).to_json
86
+ @logger.debug "data = " + data
87
+ @logger.debug "params = " + params.inspect
88
+ @logger.debug "options = " + options.inspect
89
+ token = params["oauth"]
90
+ parse_response RestClient.post(url(method) + "?oauth="+token, {:data => data, :file => file}, :content_type => 'application/json', :accept => :json), options
91
+ #parse_response(RestClient.post(append_params(url(method), add_params(method, params)), {:data => data, :file => file}, :content_type => 'application/json'), options)
69
92
  rescue RestClient::Exception => ex
70
93
  process_ex(ex)
71
94
  end
72
95
  end
73
96
 
74
97
  def post(method, params={}, options={})
98
+ @logger.debug "params = " + params.inspect
99
+ @logger.debug "options = " + options.inspect
100
+ @logger.debug "params.payload = " + params[:payload].inspect
101
+ token = params["token"]
102
+ @logger.debug "token = "+ token.inspect
75
103
  begin
76
- parse_response RestClient.post(url(method), add_params(method, params).to_json, headers), options
104
+ # here's what get() does:
105
+ #parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
106
+ parse_response(RestClient.post(url(method)+"?oauth="+token, add_params(method, params).to_json, headers.merge!({:content_type=>'application/json', :accept => "json"})), options)
107
+ # was , add_params(method, params).to_json, headers.merge!({:content_type=>'application/json'})), options)
77
108
  #ClientHelper.run_http(host, access_key, secret_key, :post, method, nil, params)
78
109
  rescue RestClient::Exception => ex
79
110
  process_ex(ex)
@@ -98,23 +129,10 @@ module SimpleWorker
98
129
  end
99
130
  end
100
131
 
101
- def url(command_path)
102
- url = host + command_path
103
- url
104
- end
105
-
106
132
  def add_params(command_path, hash)
107
- v = version||"0.1"
133
+ v = version || "2.0"
108
134
  ts = SimpleWorker::Api::Signatures.generate_timestamp(Time.now.gmtime)
109
- # puts 'timestamp = ' + ts
110
- sig = case v
111
- when "0.2"
112
- SimpleWorker::Api::Signatures.generate_signature(command_path + SimpleWorker::Api::Signatures.hash_to_s(hash), ts, secret_key)
113
- when "0.1"
114
- SimpleWorker::Api::Signatures.generate_signature(command_path, ts, secret_key)
115
- end
116
-
117
- extra_params = {'sigv'=>v, 'sig' => sig, 'timestamp' => ts, 'access_key' => access_key}
135
+ extra_params = {'version'=>v, 'timestamp' => ts, 'oauth' => token}
118
136
  hash.merge!(extra_params)
119
137
  end
120
138
 
@@ -122,8 +140,9 @@ module SimpleWorker
122
140
  host += "?"
123
141
  i = 0
124
142
  params.each_pair do |k, v|
143
+ #puts "k=#{k} v=#{v}"
125
144
  host += "&" if i > 0
126
- host += k + "=" + CGI.escape(v)
145
+ host += k + "=" + (v.is_a?(String) ? CGI.escape(v) : v.to_s)
127
146
  i +=1
128
147
  end
129
148
  return host
@@ -135,12 +154,12 @@ module SimpleWorker
135
154
  end
136
155
 
137
156
  def parse_response(response, options={})
138
- # puts 'PARSE RESPONSE!'
157
+ #puts 'PARSE RESPONSE: ' + response.to_s
139
158
  unless options[:parse] == false
140
159
  begin
141
160
  return JSON.parse(response.to_s)
142
161
  rescue => ex
143
- puts 'response that caused error = ' + response.to_s
162
+ puts 'parse_response: response that caused error = ' + response.to_s
144
163
  raise ex
145
164
  end
146
165
  else
@@ -280,8 +280,8 @@ module SimpleWorker
280
280
  end
281
281
 
282
282
  # Retrieves the log for this worker from the SimpleWorker service.
283
- def get_log
284
- SimpleWorker.service.log(task_id)
283
+ def get_log(options={})
284
+ SimpleWorker.service.get_log(task_id, options)
285
285
  end
286
286
 
287
287
  # Callbacks for developer
@@ -407,7 +407,7 @@ module SimpleWorker
407
407
  #merged.uniq!
408
408
  # merged_mailers.uniq!
409
409
  options_for_upload = {:merge=>merged, :unmerge=>unmerged, :merged_gems=>merged_gems, :merged_mailers=>merged_mailers, :merged_folders=>merged_folders}
410
- options_for_upload[:name] = options[:name] if options[:name]
410
+ options_for_upload.merge!(options)
411
411
  SimpleWorker.service.upload(rfile, subclass.name, options_for_upload)
412
412
  self.class.instance_variable_set(:@uploaded, true)
413
413
  else
@@ -10,9 +10,10 @@ module SimpleWorker
10
10
  # config.database configures a database connection. If specified like ActiveRecord, SimpleWorker will automatically establish a connection
11
11
  # for you before running your worker.
12
12
  class Config
13
- attr_accessor :access_key,
14
- :secret_key,
13
+ attr_accessor :token,
14
+ :project_id,
15
15
  :host,
16
+ :port,
16
17
  :global_attributes,
17
18
  :models,
18
19
  :mailers,
@@ -130,6 +131,8 @@ module SimpleWorker
130
131
  # gem_info[:no_require] = true
131
132
  end
132
133
  end
134
+ else
135
+ SimpleWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
133
136
  end
134
137
  # else
135
138
  # SimpleWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
@@ -148,8 +151,8 @@ module SimpleWorker
148
151
 
149
152
  def get_atts_to_send
150
153
  config_data = {}
151
- config_data['access_key'] = access_key
152
- config_data['secret_key'] = secret_key
154
+ config_data['token'] = token
155
+ config_data['project_id'] = project_id
153
156
  config_data['database'] = self.database if self.database
154
157
  config_data['mailer'] = self.mailer if self.mailer
155
158
  config_data['global_attributes'] = self.global_attributes if self.global_attributes
@@ -241,7 +244,7 @@ module SimpleWorker
241
244
  path = SimpleWorker::Service.get_gem_path(gem_info)
242
245
  SimpleWorker.logger.debug "Gem path=#{path}"
243
246
  if !path
244
- raise "Gem path not found for #{gem_name}"
247
+ raise "Gem '#{gem_name}' not found."
245
248
  end
246
249
  gem_info[:path] = path
247
250
  gem_info
@@ -17,9 +17,9 @@ def init_mailer(sw_config)
17
17
  mailer_config = sw_config['mailer']
18
18
  if mailer_config
19
19
  require 'action_mailer'
20
- ActionMailer::Base.raise_delivery_errors = true
21
- ActionMailer::Base.smtp_settings = mailer_config
22
- ActionMailer::Base.delivery_method = :smtp
20
+ ActionMailer::Base.raise_delivery_errors = true
21
+ ActionMailer::Base.smtp_settings = mailer_config
22
+ ActionMailer::Base.delivery_method = :smtp
23
23
  end
24
24
  end
25
25
  end
@@ -61,10 +61,11 @@ end
61
61
  def init_worker_service_for_runner(job_data)
62
62
  SimpleWorker.configure do |config|
63
63
  sw_config = job_data['sw_config']
64
- config.access_key = sw_config['access_key']
65
- config.secret_key = sw_config['secret_key']
64
+ config.token = sw_config['token']
65
+ config.project_id = sw_config['project_id']
66
66
  #puts 'Setting host to ' + host.inspect
67
67
  config.host = sw_config['host'] if sw_config['host']
68
+ config.host = sw_config['port'] if sw_config['port']
68
69
  db_config = sw_config['database']
69
70
  if db_config
70
71
  config.database = db_config
@@ -77,22 +78,22 @@ def init_worker_service_for_runner(job_data)
77
78
  end
78
79
  end
79
80
 
80
-
81
+ # SimpleWorker.logger.level == Logger::DEBUG
81
82
  run_data = JSON.load(File.open(run_data_file))
82
83
  # Load in job data
83
84
  job_data = JSON.load(File.open(job_data_file))
84
85
  job_data.merge!(run_data)
85
- puts 'job_data=' + job_data.inspect
86
+ SimpleWorker.logger.debug 'job_data=' + job_data.inspect
86
87
 
87
88
  sw_config = job_data['sw_config']
88
- init_database_connection(sw_config)
89
- init_mailer(sw_config)
90
- SimpleWorker.disable_queueing()
91
- runner_class = get_class_to_run(job_data['class_name'])
92
- SimpleWorker.running_class = runner_class
93
- runner = init_runner(runner_class, job_data, dirname)
94
- init_worker_service_for_runner(job_data)
95
- SimpleWorker.enable_queueing()
89
+ init_database_connection(sw_config)
90
+ init_mailer(sw_config)
91
+ SimpleWorker.disable_queueing()
92
+ runner_class = get_class_to_run(job_data['class_name'])
93
+ SimpleWorker.running_class = runner_class
94
+ runner = init_runner(runner_class, job_data, dirname)
95
+ init_worker_service_for_runner(job_data)
96
+ SimpleWorker.enable_queueing()
96
97
 
97
98
  # Let's run it!
98
- runner_return_data = runner.run
99
+ runner_return_data = runner.run
@@ -12,18 +12,20 @@ module SimpleWorker
12
12
 
13
13
  attr_accessor :config
14
14
 
15
- def initialize(access_key, secret_key, options={})
15
+ def initialize(token, options={})
16
16
  if options[:config]
17
17
  self.config = options[:config]
18
18
  else
19
19
  c = SimpleWorker::Config.new unless self.config
20
- c.access_key = access_key
21
- c.secret_key = secret_key
20
+ c.token = token
22
21
  self.config = c
23
22
  end
24
- super("http://api.simpleworker.com/api/", access_key, secret_key, options)
23
+ options[:version] = SimpleWorker.api_version
24
+ options[:logger] = SimpleWorker.logger
25
+ super("api2.simpleworker.com", token, options)
25
26
  self.host = self.config.host if self.config && self.config.host
26
27
  SimpleWorker.logger.info 'SimpleWorker initialized.'
28
+ SimpleWorker.logger.debug ' host = ' + self.host.inspect
27
29
  end
28
30
 
29
31
  # Options:
@@ -31,10 +33,11 @@ module SimpleWorker
31
33
  # - :merge => array of files to merge in with this file
32
34
  def upload(filename, class_name, options={})
33
35
  name = options[:name] || class_name
36
+ project_id = get_project_id(options)
34
37
  # puts "Uploading #{class_name}"
35
38
  # check whether it should upload again
36
39
  tmp = Dir.tmpdir()
37
- md5file = "simple_worker_#{class_name.gsub("::", ".")}_#{access_key[0, 8]}.md5"
40
+ md5file = "simple_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
38
41
  existing_md5 = nil
39
42
  md5_f = File.join(tmp, md5file)
40
43
  if File.exists?(md5_f)
@@ -61,7 +64,7 @@ module SimpleWorker
61
64
  upload_code(name, zip_filename, 'runner.rb', :runtime=>'ruby')
62
65
  end
63
66
 
64
- rescue => ex
67
+ rescue Exception => ex
65
68
  # if it errors, let's delete md5 since it wouldn't have uploaded.
66
69
  File.delete(md5_f)
67
70
  raise ex
@@ -84,13 +87,12 @@ module SimpleWorker
84
87
  gems= Gem::Specification.respond_to?(:each) ? Gem::Specification.find_all_by_name(gem_name) : Gem::GemPathSearcher.new.find_all(gem_name)
85
88
  # gems = searcher.init_gemspecs.select { |gem| gem.name==gem_name }
86
89
  gems = Gem::GemPathSearcher.new.init_gemspecs.select { |gem| gem.name==gem_name } if !gems || gems.empty?
87
- logger.debug 'gems found=' + gems.inspect
90
+ SimpleWorker.logger.debug 'gems found=' + gems.inspect
88
91
  gems = gems.select { |g| g.version.version==gem_info[:version] } if gem_info[:version]
89
92
  if !gems.empty?
90
93
  gem = gems.first
91
94
  gem.full_gem_path
92
95
  else
93
- SimpleWorker.logger.warn "Gem file was not found for #{gem_name}, continuing anyways."
94
96
  return nil
95
97
  end
96
98
  end
@@ -251,7 +253,7 @@ end
251
253
  merged_gems.each_pair do |k, gem|
252
254
  next unless gem[:merge]
253
255
  # puts 'gem=' + gem.inspect
254
- path = gem[:path] # get_gem_path(gem)
256
+ path = gem[:path]
255
257
  if path
256
258
  SimpleWorker.logger.debug "Collecting gem #{path}"
257
259
  paths_to_use = ["#{path}/*", "#{path}/lib/**/**"]
@@ -334,17 +336,24 @@ end
334
336
  SimpleWorker.logger.info 'file size to upload: ' + File.size(package_file).to_s
335
337
  options = {
336
338
  "name"=>name,
337
- "class_name"=>name, # todo: remove this shortly
338
339
  "standalone"=>true,
339
- "runtime"=>options[:runtime],
340
+ "runtime"=>options[:runtime] || "ruby",
340
341
  "file_name"=> exec_file # File.basename(filename)
341
342
  }
342
343
  #puts 'options for upload=' + options.inspect
343
344
  SimpleWorker.logger.info "Uploading now..."
344
- ret = post_file("code/put", File.new(package_file), options)
345
+ ret = post_file("#{project_url_prefix(get_project_id(options))}workers", File.new(package_file), options)
345
346
  SimpleWorker.logger.info "Done uploading."
346
347
  return ret
348
+ end
347
349
 
350
+ def project_url_prefix(project_id = 0)
351
+ # SimpleWorker.logger.info "project_url_prefix, project_id = " + project_id.inspect
352
+ if project_id == 0
353
+ return false
354
+ project_id = config.project_id
355
+ end
356
+ "projects/#{project_id}/"
348
357
  end
349
358
 
350
359
  def wait_until_complete(task_id)
@@ -364,13 +373,12 @@ end
364
373
 
365
374
  def add_sw_params(hash_to_send)
366
375
  # todo: remove secret key?? Can use worker service from within a worker without it now
367
- hash_to_send["sw_access_key"] = self.access_key
368
- hash_to_send["sw_secret_key"] = self.secret_key
376
+ hash_to_send["token"] = self.token
369
377
  hash_to_send["api_version"] = SimpleWorker.api_version
370
378
  end
371
379
 
372
380
  def check_config
373
- if self.config.nil? || self.config.access_key.nil?
381
+ if self.config.nil? || self.config.token.nil?
374
382
  raise "Invalid SimpleWorker configuration, no access key specified."
375
383
  end
376
384
  end
@@ -387,6 +395,7 @@ end
387
395
  if !data.is_a?(Array)
388
396
  data = [data]
389
397
  end
398
+ # Now we need to add class_name to the payload
390
399
  data.each do |d|
391
400
  d['class_name'] = class_name
392
401
  d['access_key'] = class_name
@@ -396,7 +405,6 @@ end
396
405
  hash_to_send["payload"] = data
397
406
  hash_to_send["class_name"] = class_name
398
407
  hash_to_send["name"] = name
399
- #hash_to_send["standalone"] = true # new school
400
408
  hash_to_send["priority"] = options[:priority] if options[:priority]
401
409
  hash_to_send["options"] = options
402
410
  add_sw_params(hash_to_send)
@@ -404,16 +412,17 @@ end
404
412
  # todo: REMOVE THIS
405
413
  hash_to_send["rails_env"] = RAILS_ENV
406
414
  end
407
- return queue_raw(class_name, hash_to_send)
408
-
415
+ return queue_raw(class_name, hash_to_send, options)
409
416
  end
410
417
 
411
- def queue_raw(class_name, data={})
418
+ def queue_raw(class_name, data={}, options={})
412
419
  params = nil
413
420
  hash_to_send = data
414
421
  hash_to_send["class_name"] = class_name unless hash_to_send["class_name"]
415
422
  hash_to_send["name"] = class_name unless hash_to_send["name"]
416
- ret = post("queue/add", hash_to_send)
423
+ uri = project_url_prefix(get_project_id(options)) + "jobs"
424
+ SimpleWorker.logger.info 'queue_raw , uri = ' + uri
425
+ ret = post(uri, hash_to_send)
417
426
  ret
418
427
  end
419
428
 
@@ -451,15 +460,61 @@ end
451
460
  ret
452
461
  end
453
462
 
454
- def get_schedules()
463
+ def get_projects()
464
+ hash_to_send = {}
465
+ ret = get("projects", hash_to_send)
466
+ ret
467
+ end
468
+
469
+ def get_project_id(options={})
470
+ options[:project_id] || config.project_id
471
+ end
472
+
473
+ def get_project(options={})
474
+ hash_to_send = {}
475
+
476
+ ret = get("projects/"+ get_project_id(options) +"/", hash_to_send)
477
+ #uri = project_url_prefix(id)
478
+ #puts "get_project, uri = " + uri
479
+ #ret = get(uri, hash_to_send)
480
+ ret
481
+ end
482
+
483
+ def get_workers(options={})
484
+ hash_to_send = {}
485
+ uri = "projects/" + get_project_id(options) + "/workers/"
486
+ ret = get(uri, hash_to_send)
487
+ ret
488
+ end
489
+
490
+ def get_schedules(options={})
491
+ hash_to_send = {}
492
+ uri = "projects/" + get_project_id(options) + "/schedules/"
493
+ ret = get(uri, hash_to_send)
494
+ ret
495
+ end
496
+
497
+ def get_jobs(options={})
455
498
  hash_to_send = {}
456
- ret = get("scheduler/list", hash_to_send)
499
+ uri = "projects/" + get_project_id(options) + "/jobs/"
500
+ ret = get(uri, hash_to_send)
501
+ ret
502
+ end
503
+
504
+ def get_log(job_id, options={})
505
+ log(job_id, options)
506
+ end
507
+
508
+ def log(task_id, options={})
509
+ data = {}
510
+ ret = get("#{project_url_prefix(get_project_id(options))}jobs/#{task_id}/log", data, :parse=>false)
457
511
  ret
458
512
  end
459
513
 
460
- def status(task_id)
514
+
515
+ def status(task_id, options={})
461
516
  data = {"task_id"=>task_id}
462
- ret = get("task/status", data)
517
+ ret = get("#{project_url_prefix(get_project_id(options))}jobs/#{task_id}", data)
463
518
  ret
464
519
  end
465
520
 
@@ -475,14 +530,6 @@ end
475
530
  post("task/setstatus", data)
476
531
  end
477
532
 
478
- def log(task_id)
479
- data = {"task_id"=>task_id}
480
- ret = get("task/log", data, {:parse=>false})
481
- # puts ' ret=' + ret.inspect
482
- # ret["log"] = Base64.decode64(ret["log"])
483
- ret
484
- end
485
-
486
533
 
487
534
  end
488
535
 
data/lib/simple_worker.rb CHANGED
@@ -16,8 +16,10 @@ module SimpleWorker
16
16
 
17
17
  def configure()
18
18
  yield(config)
19
- if config && config.access_key && config.secret_key
20
- SimpleWorker.service ||= Service.new(config.access_key, config.secret_key, :config=>config)
19
+ if config && config.token
20
+ SimpleWorker.service ||= Service.new(config.token, :config=>config)
21
+ else
22
+ @@logger.warn "No token specified in configure, be sure to set it!"
21
23
  end
22
24
  end
23
25
 
@@ -30,7 +32,7 @@ module SimpleWorker
30
32
  end
31
33
 
32
34
  def api_version
33
- 3
35
+ 2
34
36
  end
35
37
  end
36
38
 
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: simple_worker
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease: 6
5
- version: 2.0.0.beta.3
5
+ version: 2.0.0.beta.4
6
6
  platform: ruby
7
7
  authors:
8
8
  - Travis Reeder