simple_worker 2.0.0.beta.3 → 2.0.0.beta.4

data/README.markdown

@@ -13,11 +13,11 @@ Install SimpleWorker Gem
 Configure SimpleWorker
 ----------------------
 
-You really just need your access keys.
+You really just need your token, which you can get [here][2]
+[2]: http://simpleworker.com/tokens 
 
     SimpleWorker.configure do |config|
-        config.access_key = ACCESS_KEY
-        config.secret_key = SECRET_KEY
+        config.token = TOKEN
     end
     
 Generate a Worker with Rails

data/lib/simple_worker/api.rb

@@ -30,24 +30,28 @@ module SimpleWorker
     #  host: endpoint url for service
     class Client
 
-      @@logger = Logger.new(STDOUT)
-      @@logger.level = Logger::INFO
+      attr_accessor :host, :port, :token, :version, :config
 
-      def self.logger
-        @@logger
-      end
+      def initialize(host, token, options={})
+        @host = host
+        @config = options[:config]
+        @port = options[:port] || @config.port || 80
+        @token = token
+        @version = options[:version]
+        @logger = options[:logger]
 
-      attr_accessor :host, :access_key, :secret_key, :version
+      end
 
-      def initialize(host, access_key, secret_key, options={})
-        @host = host
-        @access_key = access_key
-        @secret_key = secret_key
+      def url(command_path)
+        url = "http://#{host}:#{port}/#{@version}/#{command_path}"
+        # @logger.debug "url: " + url.to_s
+        url
       end
 
       def process_ex(ex)
         body = ex.http_body
-        @@logger.debug 'EX BODY=' + body.to_s
+        @logger.debug 'EX http_code: ' + ex.http_code.to_s
+        @logger.debug 'EX BODY=' + body.to_s
         decoded_ex = JSON.parse(body)
         exception = Exception.new(ex.message + ": " + decoded_ex["msg"])
         exception.set_backtrace(decoded_ex["backtrace"].split(",")) if decoded_ex["backtrace"]
@@ -55,25 +59,52 @@ module SimpleWorker
       end
 
       def get(method, params={}, options={})
-        begin
+        #begin
 #                ClientHelper.run_http(host, access_key, secret_key, :get, method, nil, params)
-          parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
-        rescue RestClient::Exception  => ex
-          process_ex(ex)
-        end
+          full_url = url(method)
+          all_params = add_params(method,params)
+
+          url_plus_params = append_params(full_url, all_params)
+          resp = RestClient.get(url_plus_params, headers)
+
+          parse_response(resp, options)
+        
+          # Was: 
+          #parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
+        #rescue RestClient::Exception  => ex
+        #  process_ex(ex)
+        #end
       end
 
       def post_file(method, file, params={}, options={})
         begin
-          parse_response RestClient.post(url(method), add_params(method, params).merge!({:file=>file}), :multipart => true), options
+          #params.delete("runtime")
+          #params["runtime"]='ruby'
+          #params.delete("file_name")
+          #params["file_name"] = "runner.rb"
+          data = add_params(method, params).to_json
+          @logger.debug "data = " + data
+          @logger.debug "params = " + params.inspect
+          @logger.debug "options = " + options.inspect
+          token = params["oauth"]
+          parse_response RestClient.post(url(method) + "?oauth="+token, {:data => data, :file => file}, :content_type => 'application/json', :accept => :json), options
+          #parse_response(RestClient.post(append_params(url(method), add_params(method, params)), {:data => data, :file => file}, :content_type => 'application/json'), options)
         rescue RestClient::Exception  => ex
           process_ex(ex)
         end
       end
 
       def post(method, params={}, options={})
+          @logger.debug "params = " + params.inspect
+          @logger.debug "options = " + options.inspect
+          @logger.debug "params.payload = " + params[:payload].inspect
+          token = params["token"]
+          @logger.debug "token = "+ token.inspect
         begin
-          parse_response RestClient.post(url(method), add_params(method, params).to_json, headers), options
+          # here's what get() does:
+          #parse_response RestClient.get(append_params(url(method), add_params(method, params)), headers), options
+          parse_response(RestClient.post(url(method)+"?oauth="+token, add_params(method, params).to_json, headers.merge!({:content_type=>'application/json', :accept => "json"})), options)
+          # was , add_params(method, params).to_json, headers.merge!({:content_type=>'application/json'})), options)
           #ClientHelper.run_http(host, access_key, secret_key, :post, method, nil, params)
         rescue RestClient::Exception  => ex
           process_ex(ex)
@@ -98,23 +129,10 @@ module SimpleWorker
         end
       end
 
-      def url(command_path)
-        url = host + command_path
-        url
-      end
-
       def add_params(command_path, hash)
-        v = version||"0.1"
+        v = version || "2.0"
         ts = SimpleWorker::Api::Signatures.generate_timestamp(Time.now.gmtime)
-        # puts 'timestamp = ' + ts
-        sig = case v
-                when "0.2"
-                  SimpleWorker::Api::Signatures.generate_signature(command_path + SimpleWorker::Api::Signatures.hash_to_s(hash), ts, secret_key)
-                when "0.1"
-                  SimpleWorker::Api::Signatures.generate_signature(command_path, ts, secret_key)
-              end
-
-        extra_params = {'sigv'=>v, 'sig' => sig, 'timestamp' => ts, 'access_key' => access_key}
+        extra_params = {'version'=>v, 'timestamp' => ts, 'oauth' => token}
         hash.merge!(extra_params)
       end
 
@@ -122,8 +140,9 @@ module SimpleWorker
         host += "?"
         i = 0
         params.each_pair do |k, v|
+          #puts "k=#{k} v=#{v}"
           host += "&" if i > 0
-          host += k + "=" + CGI.escape(v)
+          host += k + "=" + (v.is_a?(String) ? CGI.escape(v) : v.to_s)
           i +=1
         end
         return host
@@ -135,12 +154,12 @@ module SimpleWorker
       end
 
       def parse_response(response, options={})
-#        puts 'PARSE RESPONSE!'
+        #puts 'PARSE RESPONSE: ' + response.to_s
         unless options[:parse] == false
           begin
             return JSON.parse(response.to_s)
           rescue => ex
-            puts 'response that caused error = ' + response.to_s
+            puts 'parse_response: response that caused error = ' + response.to_s
             raise ex
           end
         else

data/lib/simple_worker/base.rb

@@ -280,8 +280,8 @@ module SimpleWorker
     end
 
     # Retrieves the log for this worker from the SimpleWorker service.
-    def get_log
-      SimpleWorker.service.log(task_id)
+    def get_log(options={})
+      SimpleWorker.service.get_log(task_id, options)
     end
 
     # Callbacks for developer
@@ -407,7 +407,7 @@ module SimpleWorker
 #merged.uniq!
 #        merged_mailers.uniq!
         options_for_upload = {:merge=>merged, :unmerge=>unmerged, :merged_gems=>merged_gems, :merged_mailers=>merged_mailers, :merged_folders=>merged_folders}
-        options_for_upload[:name] = options[:name] if options[:name]
+        options_for_upload.merge!(options)
         SimpleWorker.service.upload(rfile, subclass.name, options_for_upload)
         self.class.instance_variable_set(:@uploaded, true)
       else

data/lib/simple_worker/config.rb

@@ -10,9 +10,10 @@ module SimpleWorker
   # config.database configures a database connection. If specified like ActiveRecord, SimpleWorker will automatically establish a connection
   # for you before running your worker.
   class Config
-    attr_accessor :access_key,
-                  :secret_key,
+    attr_accessor :token,
+                  :project_id,
                   :host,
+                  :port,
                   :global_attributes,
                   :models,
                   :mailers,
@@ -130,6 +131,8 @@ module SimpleWorker
 #              gem_info[:no_require] = true
             end
           end
+        else
+           SimpleWorker.logger.warn "Could not find '#{gem_info[:name]}' specified in Bundler, continuing anyways."
         end
 #        else
 #          SimpleWorker.logger.warn "Could not find gem spec for #{gem_info[:name]}"
@@ -148,8 +151,8 @@ module SimpleWorker
 
     def get_atts_to_send
       config_data = {}
-      config_data['access_key'] = access_key
-      config_data['secret_key'] = secret_key
+      config_data['token'] = token
+      config_data['project_id'] = project_id
       config_data['database'] = self.database if self.database
       config_data['mailer'] = self.mailer if self.mailer
       config_data['global_attributes'] = self.global_attributes if self.global_attributes
@@ -241,7 +244,7 @@ module SimpleWorker
       path = SimpleWorker::Service.get_gem_path(gem_info)
       SimpleWorker.logger.debug "Gem path=#{path}"
       if !path
-        raise "Gem path not found for #{gem_name}"
+        raise "Gem '#{gem_name}' not found."
       end
       gem_info[:path] = path
       gem_info

data/lib/simple_worker/server/runner.rb

@@ -17,9 +17,9 @@ def init_mailer(sw_config)
     mailer_config = sw_config['mailer']
     if mailer_config
       require 'action_mailer'
-       ActionMailer::Base.raise_delivery_errors = true
-       ActionMailer::Base.smtp_settings =  mailer_config
-       ActionMailer::Base.delivery_method = :smtp
+      ActionMailer::Base.raise_delivery_errors = true
+      ActionMailer::Base.smtp_settings = mailer_config
+      ActionMailer::Base.delivery_method = :smtp
     end
   end
 end
@@ -61,10 +61,11 @@ end
 def init_worker_service_for_runner(job_data)
   SimpleWorker.configure do |config|
     sw_config = job_data['sw_config']
-    config.access_key = sw_config['access_key']
-    config.secret_key = sw_config['secret_key']
+    config.token = sw_config['token']
+    config.project_id = sw_config['project_id']
     #puts 'Setting host to ' + host.inspect
     config.host = sw_config['host'] if sw_config['host']
+    config.host = sw_config['port'] if sw_config['port']
     db_config = sw_config['database']
     if db_config
       config.database = db_config
@@ -77,22 +78,22 @@ def init_worker_service_for_runner(job_data)
   end
 end
 
-
+# SimpleWorker.logger.level ==  Logger::DEBUG
 run_data = JSON.load(File.open(run_data_file))
 # Load in job data
 job_data = JSON.load(File.open(job_data_file))
 job_data.merge!(run_data)
-puts 'job_data=' + job_data.inspect
+SimpleWorker.logger.debug 'job_data=' + job_data.inspect
 
 sw_config = job_data['sw_config']
-  init_database_connection(sw_config)
-  init_mailer(sw_config)
-  SimpleWorker.disable_queueing()
-  runner_class = get_class_to_run(job_data['class_name'])
-  SimpleWorker.running_class = runner_class
-  runner = init_runner(runner_class, job_data, dirname)
-  init_worker_service_for_runner(job_data)
-  SimpleWorker.enable_queueing()
+init_database_connection(sw_config)
+init_mailer(sw_config)
+SimpleWorker.disable_queueing()
+runner_class = get_class_to_run(job_data['class_name'])
+SimpleWorker.running_class = runner_class
+runner = init_runner(runner_class, job_data, dirname)
+init_worker_service_for_runner(job_data)
+SimpleWorker.enable_queueing()
 
 # Let's run it!
-  runner_return_data = runner.run
+runner_return_data = runner.run

data/lib/simple_worker/service.rb

@@ -12,18 +12,20 @@ module SimpleWorker
 
     attr_accessor :config
 
-    def initialize(access_key, secret_key, options={})
+    def initialize(token, options={})
       if options[:config]
         self.config = options[:config]
       else
         c = SimpleWorker::Config.new unless self.config
-        c.access_key = access_key
-        c.secret_key = secret_key
+        c.token = token
         self.config = c
       end
-      super("http://api.simpleworker.com/api/", access_key, secret_key, options)
+      options[:version] = SimpleWorker.api_version
+      options[:logger] = SimpleWorker.logger
+      super("api2.simpleworker.com", token, options)
       self.host = self.config.host if self.config && self.config.host
       SimpleWorker.logger.info 'SimpleWorker initialized.'
+      SimpleWorker.logger.debug ' host = ' + self.host.inspect
     end
 
     # Options:
@@ -31,10 +33,11 @@ module SimpleWorker
     #    - :merge => array of files to merge in with this file
     def upload(filename, class_name, options={})
       name = options[:name] || class_name
+      project_id = get_project_id(options)
 #      puts "Uploading #{class_name}"
 # check whether it should upload again
       tmp = Dir.tmpdir()
-      md5file = "simple_worker_#{class_name.gsub("::", ".")}_#{access_key[0, 8]}.md5"
+      md5file = "simple_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
       existing_md5 = nil
       md5_f = File.join(tmp, md5file)
       if File.exists?(md5_f)
@@ -61,7 +64,7 @@ module SimpleWorker
           upload_code(name, zip_filename, 'runner.rb', :runtime=>'ruby')
         end
 
-      rescue => ex
+      rescue Exception => ex
         # if it errors, let's delete md5 since it wouldn't have uploaded.
         File.delete(md5_f)
         raise ex
@@ -84,13 +87,12 @@ module SimpleWorker
       gems= Gem::Specification.respond_to?(:each) ? Gem::Specification.find_all_by_name(gem_name) : Gem::GemPathSearcher.new.find_all(gem_name)
       #      gems     = searcher.init_gemspecs.select { |gem| gem.name==gem_name }
       gems = Gem::GemPathSearcher.new.init_gemspecs.select { |gem| gem.name==gem_name } if !gems || gems.empty?
-      logger.debug 'gems found=' + gems.inspect
+      SimpleWorker.logger.debug 'gems found=' + gems.inspect
       gems = gems.select { |g| g.version.version==gem_info[:version] } if gem_info[:version]
       if !gems.empty?
         gem = gems.first
         gem.full_gem_path
       else
-        SimpleWorker.logger.warn "Gem file was not found for #{gem_name}, continuing anyways."
         return nil
       end
     end
@@ -251,7 +253,7 @@ end
           merged_gems.each_pair do |k, gem|
             next unless gem[:merge]
 #            puts 'gem=' + gem.inspect
-            path = gem[:path] # get_gem_path(gem)
+            path = gem[:path]
             if path
               SimpleWorker.logger.debug "Collecting gem #{path}"
               paths_to_use = ["#{path}/*", "#{path}/lib/**/**"]
@@ -334,17 +336,24 @@ end
       SimpleWorker.logger.info 'file size to upload: ' + File.size(package_file).to_s
       options = {
           "name"=>name,
-          "class_name"=>name, # todo: remove this shortly
           "standalone"=>true,
-          "runtime"=>options[:runtime],
+          "runtime"=>options[:runtime] || "ruby",
           "file_name"=> exec_file # File.basename(filename)
       }
       #puts 'options for upload=' + options.inspect
       SimpleWorker.logger.info "Uploading now..."
-      ret = post_file("code/put", File.new(package_file), options)
+      ret = post_file("#{project_url_prefix(get_project_id(options))}workers", File.new(package_file), options)
       SimpleWorker.logger.info "Done uploading."
       return ret
+    end
 
+    def project_url_prefix(project_id = 0)
+      # SimpleWorker.logger.info "project_url_prefix, project_id = " + project_id.inspect
+      if project_id == 0
+        return false
+        project_id = config.project_id
+      end
+      "projects/#{project_id}/"
     end
 
     def wait_until_complete(task_id)
@@ -364,13 +373,12 @@ end
 
     def add_sw_params(hash_to_send)
       # todo: remove secret key??  Can use worker service from within a worker without it now
-      hash_to_send["sw_access_key"] = self.access_key
-      hash_to_send["sw_secret_key"] = self.secret_key
+      hash_to_send["token"] = self.token
       hash_to_send["api_version"] = SimpleWorker.api_version
     end
 
     def check_config
-      if self.config.nil? || self.config.access_key.nil?
+      if self.config.nil? || self.config.token.nil?
         raise "Invalid SimpleWorker configuration, no access key specified."
       end
     end
@@ -387,6 +395,7 @@ end
       if !data.is_a?(Array)
         data = [data]
       end
+      # Now we need to add class_name to the payload
       data.each do |d|
         d['class_name'] = class_name
         d['access_key'] = class_name
@@ -396,7 +405,6 @@ end
       hash_to_send["payload"] = data
       hash_to_send["class_name"] = class_name
       hash_to_send["name"] = name
-      #hash_to_send["standalone"] = true # new school
       hash_to_send["priority"] = options[:priority] if options[:priority]
       hash_to_send["options"] = options
       add_sw_params(hash_to_send)
@@ -404,16 +412,17 @@ end
         # todo: REMOVE THIS
         hash_to_send["rails_env"] = RAILS_ENV
       end
-      return queue_raw(class_name, hash_to_send)
-
+      return queue_raw(class_name, hash_to_send, options)
     end
 
-    def queue_raw(class_name, data={})
+    def queue_raw(class_name, data={}, options={})
       params = nil
       hash_to_send = data
       hash_to_send["class_name"] = class_name unless hash_to_send["class_name"]
       hash_to_send["name"] = class_name unless hash_to_send["name"]
-      ret = post("queue/add", hash_to_send)
+      uri = project_url_prefix(get_project_id(options)) + "jobs"
+      SimpleWorker.logger.info 'queue_raw , uri = ' + uri
+      ret = post(uri, hash_to_send)
       ret
     end
 
@@ -451,15 +460,61 @@ end
       ret
     end
 
-    def get_schedules()
+    def get_projects()
+      hash_to_send = {}
+      ret = get("projects", hash_to_send)
+      ret
+    end
+
+    def get_project_id(options={})
+      options[:project_id] || config.project_id
+    end
+
+    def get_project(options={})
+      hash_to_send = {}
+
+      ret = get("projects/"+ get_project_id(options) +"/", hash_to_send)
+      #uri = project_url_prefix(id)
+      #puts "get_project, uri = " + uri
+      #ret = get(uri, hash_to_send)
+      ret
+    end
+
+    def get_workers(options={})
+      hash_to_send = {}
+      uri = "projects/" + get_project_id(options) + "/workers/"
+      ret = get(uri, hash_to_send)
+      ret
+    end
+
+    def get_schedules(options={})
+      hash_to_send = {}
+      uri = "projects/" + get_project_id(options) + "/schedules/"
+      ret = get(uri, hash_to_send)
+      ret
+    end
+
+    def get_jobs(options={})
       hash_to_send = {}
-      ret = get("scheduler/list", hash_to_send)
+      uri = "projects/" + get_project_id(options) + "/jobs/"
+      ret = get(uri, hash_to_send)
+      ret
+    end
+
+    def get_log(job_id, options={})
+      log(job_id, options)
+    end
+
+    def log(task_id, options={})
+      data = {}
+      ret = get("#{project_url_prefix(get_project_id(options))}jobs/#{task_id}/log", data, :parse=>false)
       ret
     end
 
-    def status(task_id)
+
+    def status(task_id, options={})
       data = {"task_id"=>task_id}
-      ret = get("task/status", data)
+      ret = get("#{project_url_prefix(get_project_id(options))}jobs/#{task_id}", data)
       ret
     end
 
@@ -475,14 +530,6 @@ end
       post("task/setstatus", data)
     end
 
-    def log(task_id)
-      data = {"task_id"=>task_id}
-      ret = get("task/log", data, {:parse=>false})
-#            puts ' ret=' + ret.inspect
-#            ret["log"] = Base64.decode64(ret["log"])
-      ret
-    end
-
 
   end
 

data/lib/simple_worker.rb

@@ -16,8 +16,10 @@ module SimpleWorker
 
     def configure()
       yield(config)
-      if config && config.access_key && config.secret_key
-        SimpleWorker.service ||= Service.new(config.access_key, config.secret_key, :config=>config)
+      if config && config.token
+        SimpleWorker.service ||= Service.new(config.token, :config=>config)
+      else
+        @@logger.warn "No token specified in configure, be sure to set it!"
       end
     end
 
@@ -30,7 +32,7 @@ module SimpleWorker
     end
 
     def api_version
-      3
+      2
     end
   end
 

metadata

@@ -2,7 +2,7 @@
 name: simple_worker
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 2.0.0.beta.3
+  version: 2.0.0.beta.4
 platform: ruby
 authors: 
 - Travis Reeder