simple_worker 2.0.0.beta.11 → 2.0.0.beta.15

data/README.markdown

@@ -80,6 +80,14 @@ Let's say someone does something in your app and you want to send an email about
 
 This will send it off to the SimpleWorker cloud.
 
+To queue worker without uploading you could try to do following:
+
+    data[:attr_encoded] = Base64.encode64({'@to'=>'example@email.com'}.to_json)
+    data[:sw_config] = SimpleWorker.config.get_atts_to_send
+    SimpleWorker.service.queue('EmailWorker', data)
+
+
+
 Setting Priority
 ----------------------------------------------
 
@@ -261,9 +269,7 @@ You could easily merge mailers you're using in your application.
     #or
     merge_mailer 'mailer_file', {:path_to_templates=>"templates_path"}#if you're using mailer outside of rails with custom templates path
 
-if you already set auto_merge=true all your mailers already merged.
-
-Configuring a Mailer Connection
+### Configuring a Mailer Connection
 
 If you are using Rails 3,your action_mailer connection would be configured automatically from your config
 

data/lib/simple_worker/api.rb

@@ -1,6 +1,19 @@
 require 'rest_client'
+require 'patron'
 
 module SimpleWorker
+
+  class RequestError < StandardError
+    def initialize(msg, options={})
+      super(msg)
+      @options = options
+    end
+
+    def status
+      @options[:status]
+    end
+  end
+
   module Api
 
     module Signatures
@@ -30,7 +43,7 @@ module SimpleWorker
     #  host: endpoint url for service
     class Client
 
-      attr_accessor :host, :port, :token, :version, :config
+      attr_accessor :scheme, :host, :port, :token, :version, :config
 
       def initialize(host, token, options={})
         @config = options[:config]
@@ -41,10 +54,25 @@ module SimpleWorker
         @version = options[:version]
         @logger = options[:logger]
 
+        @base_url = "#{@scheme}://#{@host}:#{@port}/#{@version}"
+
+        sess = Patron::Session.new
+        sess.timeout = 10
+        sess.base_url = @base_url
+        sess.headers['User-Agent'] = 'IronMQ Ruby Client'
+        #sess.enable_debug "/tmp/patron.debug"
+        @http_sess = sess
+
+
+      end
+
+
+      def base_url
+        @base_url
       end
 
       def url(command_path)
-        url = "#{@scheme}://#{host}:#{port}/#{@version}/#{command_path}"
+        url = "#{base_url}/#{command_path}"
         # @logger.debug "url: " + url.to_s
         url
       end
@@ -59,15 +87,32 @@ module SimpleWorker
         raise exception
       end
 
+      def check_response(response, options={})
+        status = response.status
+        body = response.body
+        res = nil
+        unless options[:parse] == false
+          res = JSON.parse(body)
+        end
+        if status < 400
+
+        else
+          raise SimpleWorker::RequestError.new((res ? "#{status}: #{res["msg"]}" : "#{status} Error! parse=false so no msg"), :status=>status)
+        end
+        res || body
+      end
+
       def get(method, params={}, options={})
         full_url = url(method)
         all_params = add_params(method, params)
-
         url_plus_params = append_params(full_url, all_params)
         @logger.debug 'get url=' + url_plus_params
-        resp = RestClient.get(url_plus_params, headers)
-
-        parse_response(resp, options)
+        response = @http_sess.request(:get, url_plus_params,
+                                      {},
+                                      {})
+        check_response(response, options)
+        body = response.body
+        parse_response(body, options)
 
       end
 
@@ -79,9 +124,9 @@ module SimpleWorker
           @logger.debug "data = " + data
           @logger.debug "params = " + params.inspect
           @logger.debug "options = " + options.inspect
-          parse_response RestClient.post(url, {:data => data, :file => file}, :content_type => 'application/json', :accept => :json), options
-            #parse_response(RestClient.post(append_params(url(method), add_params(method, params)), {:data => data, :file => file}, :content_type => 'application/json'), options)
-        rescue RestClient::Exception => ex
+          # todo: replace with patron
+          parse_response(RestClient.post(append_params(url(method), add_params(method, params)), {:data => data, :file => file}, :content_type => 'application/json'), options)
+        rescue Exception => ex
           process_ex(ex)
         end
       end
@@ -94,8 +139,17 @@ module SimpleWorker
         begin
           url = url(method) + "?oauth=" + token
           @logger.debug 'post url=' + url
-          parse_response(RestClient.post(url, add_params(method, params).to_json, headers.merge!({:content_type=>'application/json', :accept => "json"})), options)
-        rescue RestClient::Exception => ex
+          json = add_params(method, params).to_json
+          @logger.debug 'body=' + json
+          response = @http_sess.post(url, json, {"Content-Type" => 'application/json'})
+          check_response(response)
+          @logger.debug 'response: ' + response.inspect
+          body = response.body
+          parse_response(body, options)
+        rescue SimpleWorker::RequestError => ex
+          # let it throw, came from check_response
+          raise ex
+        rescue Exception => ex
           @logger.warn("Exception in post! #{ex.message}")
           @logger.warn(ex.backtrace.join("\n"))
           process_ex(ex)
@@ -105,6 +159,7 @@ module SimpleWorker
 
       def put(method, body, options={})
         begin
+          # todo: replace with patron
           parse_response RestClient.put(url(method), add_params(method, body).to_json, headers), options
         rescue RestClient::Exception => ex
           process_ex(ex)
@@ -113,6 +168,7 @@ module SimpleWorker
 
       def delete(method, params={}, options={})
         begin
+          # todo: replace with patron
           parse_response RestClient.delete(append_params(url(method), add_params(method, params))), options
         rescue RestClient::Exception => ex
           process_ex(ex)

data/lib/simple_worker/base.rb

@@ -41,11 +41,16 @@ module SimpleWorker
       def merge_gem(gem_name, options={})
         gem_info = SimpleWorker::MergeHelper.create_gem_info(gem_name, options)
         @merged_gems[gem_name.to_s] = gem_info
-        #puts 'before require ' + (options[:require] || gem_name)
-        begin
-          require gem_info[:require]
-        rescue LoadError=>ex
-          raise "Gem #{gem_name} was found, but we could not load the file '#{options[:require] || gem_name}.rb'. You may need to use :require=>x.........."
+        reqs = gem_info[:require].is_a?(Array) ? gem_info[:require] : [gem_info[:require]]
+        reqs.each do |r|
+          r2 = "#{gem_info[:path]}/lib/#{r}"
+          begin
+            puts 'requiring ' + r2
+            require r2
+          rescue LoadError=>ex
+            SimpleWorker.logger.error "Error requiring gem #{r}: #{ex.message}"
+            raise "Gem #{gem_name} was found, but we could not load the file '#{r2}'. You may need to use :require=>x.........."
+          end
         end
       end
 
@@ -151,7 +156,7 @@ module SimpleWorker
       # puts 'run_local'
       set_auto_attributes
       init_database
-	  init_mailer
+      init_mailer
       begin
         run
       rescue => ex
@@ -163,11 +168,11 @@ module SimpleWorker
       end
     end
 
-	def init_mailer
+    def init_mailer
       if SimpleWorker.config.mailer
         require 'action_mailer'
-         ActionMailer::Base.raise_delivery_errors = true
-         ActionMailer::Base.smtp_settings = (SimpleWorker.config.mailer)
+        ActionMailer::Base.raise_delivery_errors = true
+        ActionMailer::Base.smtp_settings = (SimpleWorker.config.mailer)
       end
     end
 
@@ -212,9 +217,8 @@ module SimpleWorker
       upload_if_needed(options)
 
       response = SimpleWorker.service.queue(self.class.name, sw_get_data, options)
-#            puts 'queue response=' + response.inspect
-#      @task_set_id = response["task_set_id"]
-      @task_id = response["task_id"]
+      SimpleWorker.service.logger.debug 'queue response=' + response.inspect
+      @task_id = response["tasks"][0]["id"]
       response
     end
 
@@ -270,8 +274,8 @@ module SimpleWorker
       upload_if_needed(schedule)
 
       response = SimpleWorker.service.schedule(self.class.name, sw_get_data, schedule)
-#            puts 'schedule response=' + response.inspect
-      @schedule_id = response["schedule_id"]
+      SimpleWorker.service.logger.debug 'schedule response=' + response.inspect
+      @schedule_id = response["schedules"][0]["id"]
       response
     end
 
@@ -408,6 +412,7 @@ module SimpleWorker
       self.instance_variables.each do |iv|
         payload[iv] = instance_variable_get(iv)
       end
+      data['class_name'] = self.class.name
       data[:attr_encoded] = Base64.encode64(payload.to_json)
       data[:file_name] = File.basename(self.class.instance_variable_get(:@caller_file))
       if defined?(Rails)

data/lib/simple_worker/service.rb

@@ -22,9 +22,11 @@ module SimpleWorker
       end
       options[:version] = SimpleWorker.api_version
       options[:logger] = SimpleWorker.logger
-      super("tq-aws-us-east-1.iron.io", token, options)
+      super("worker-aws-us-east-1.iron.io", token, options)
       self.host = self.config.host if self.config && self.config.host
-      self.config.merge_gem('simple_worker')#automerge simple worker gem
+      # automerge simple worker gem and dependenices
+      self.config.merge_gem('rest-client')
+      self.config.merge_gem('simple_worker')
       SimpleWorker.logger.info 'SimpleWorker initialized.'
       SimpleWorker.logger.debug ' host = ' + self.host.inspect
     end
@@ -35,8 +37,6 @@ module SimpleWorker
     def upload(filename, class_name, options={})
       name = options[:name] || class_name
       project_id = get_project_id(options)
-#      puts "Uploading #{class_name}"
-# check whether it should upload again
       tmp = Dir.tmpdir()
       md5file = "simple_worker_#{class_name.gsub("::", ".")}_#{token[0, 8]}.md5"
       existing_md5 = nil
@@ -44,7 +44,7 @@ module SimpleWorker
       if File.exists?(md5_f)
         existing_md5 = IO.read(md5_f)
       end
-# Check for code changes.
+      # Check for code changes.
       md5 = Digest::MD5.hexdigest(File.read(filename))
       new_code = false
       if self.config.force_upload || md5 != existing_md5
@@ -125,12 +125,8 @@ module SimpleWorker
       #tmp_file = File.join(Dir.tmpdir(), File.basename(filename))
       tmp_file = File.join(Dir.tmpdir(), 'runner.rb')
       File.open(tmp_file, "w") do |f|
-        File.open(File.join(File.dirname(__FILE__), "server", 'runner.rb'), 'r') do |fr|
-          while line = fr.gets
-            f.write line
-          end
-        end
-        f.write("begin\n")#error handling block start
+
+        f.write("begin\n") #error handling block start
         f.write("
 # Find environment (-e)
 dirname = ''
@@ -155,13 +151,13 @@ ARGV.each do |arg|
 end
 require 'json'
 ")
-        # require merged gems
-         merged_gems.each_pair do |k, gem|
+                           # require merged gems
+        merged_gems.each_pair do |k, gem|
           SimpleWorker.logger.debug "Bundling gem #{gem[:name]}..."
           if gem[:merge]
             f.write "$LOAD_PATH << File.join(File.dirname(__FILE__), '/gems/#{gem[:name]}/lib')\n"
           end
-#              unless gem[:no_require]
+          #              unless gem[:no_require]
           SimpleWorker.logger.debug 'writing requires: ' + gem[:require].inspect
           if gem[:require].nil?
             gem[:require] = []
@@ -173,11 +169,17 @@ require 'json'
             SimpleWorker.logger.debug 'adding require to file ' + r.to_s
             f.write "require '#{r}'\n"
           end
-#              end
-         end
+        end
+
+
+        File.open(File.join(File.dirname(__FILE__), "server", 'runner.rb'), 'r') do |fr|
+          while line = fr.gets
+            f.write line
+          end
+        end
 
         # load job data
-f.write("
+        f.write("
 # Change to user directory
 #puts 'dirname=' + dirname.inspect
 Dir.chdir(dirname)
@@ -185,11 +187,8 @@ Dir.chdir(dirname)
 job_data = JSON.load(File.open(task_data_file))
 puts 'job_data=' + job_data.inspect
 sw_config = job_data['sw_config']
-init_database_connection(sw_config)
-init_mailer(sw_config)
 ")
 
-        f.write("require 'simple_worker'\n")
 
         # add some rails stuff if using Rails
         if defined?(Rails)
@@ -204,22 +203,18 @@ end
 "
         end
 
-        if SimpleWorker.config.database && !SimpleWorker.config.database.empty?
-          f.write "require 'active_record'\n"
+        if SimpleWorker.config.extra_requires
+          SimpleWorker.config.extra_requires.each do |r|
+            f.write "require '#{r}'\n"
+          end
         end
-
         if merged_mailers && !merged_mailers.empty?
           # todo: isn't 'action_mailer already required in railtie?
           f.write "require 'action_mailer'\n"
+          f.write "init_mailer(sw_config)\n"
           f.write "ActionMailer::Base.prepend_view_path('templates')\n"
         end
-        #if SimpleWorker.config.auto_merge
-
-        if SimpleWorker.config.extra_requires
-          SimpleWorker.config.extra_requires.each do |r|
-            f.write "require '#{r}'\n"
-          end
-        end
+        f.write "init_database_connection(sw_config)"
 
         File.open(File.join(File.dirname(__FILE__), 'server', 'overrides.rb'), 'r') do |fr|
           while line = fr.gets
@@ -239,8 +234,6 @@ end
         #f.write File.open(filename, 'r') { |mo| mo.read }
         f.write("require_relative '#{File.basename(filename)}'")
 
-
-
         f.write("
   SimpleWorker.disable_queueing()
   runner_class = get_class_to_run(job_data['class_name'])
@@ -381,7 +374,7 @@ end
       "projects/#{project_id}/"
     end
 
-    def wait_until_complete(task_id)
+    def wait_until_complete(task_id, options={})
       tries = 0
       status = nil
       sleep 1
@@ -398,7 +391,7 @@ end
 
     def add_sw_params(hash_to_send)
       # todo: remove secret key??  Can use worker service from within a worker without it now
-      hash_to_send["token"] = self.token
+      hash_to_send["oauth"] = self.token
       hash_to_send["api_version"] = SimpleWorker.api_version
     end
 
@@ -408,42 +401,46 @@ end
       end
     end
 
-    def enqueue(class_name, data={}, options={})
-      queue(class_name, data, options)
+    def enqueue(name, data={}, options={})
+      queue(name, data, options)
     end
 
-    # class_name: The class name of a previously upload class, eg: MySuperWorker
+    # name: The name of previously upload worker code, eg: MySuperWorker
     # data: Arbitrary hash of your own data that your task will need to run.
-    def queue(class_name, data={}, options={})
-      puts "Queuing #{class_name}..."
+    def queue(name, data={}, options={})
+      puts "Queuing #{name}..."
       check_config
       if !data.is_a?(Array)
         data = [data]
       end
       # Now we need to add class_name to the payload
+      tasks = []
       data.each do |d|
-        d['class_name'] = class_name
+        d['class_name'] = name unless d['class_name']
+        task = {}
+        task["payload"] = d.to_json
+        task["code_name"] = name
+        task["priority"] = options[:priority] if options[:priority]
+        task["timeout"] = options[:timeout] if options[:timeout]
+        tasks << task
       end
-      name = options[:name] || class_name
+      name = options[:name] || name
       hash_to_send = {}
-      hash_to_send["payload"] = data
-      hash_to_send["class_name"] = class_name
-      hash_to_send["name"] = name
-      hash_to_send["priority"] = options[:priority] if options[:priority]
       hash_to_send["options"] = options
+      hash_to_send["tasks"] = tasks
       add_sw_params(hash_to_send)
       if defined?(RAILS_ENV)
         # todo: REMOVE THIS
         hash_to_send["rails_env"] = RAILS_ENV
       end
-      return queue_raw(class_name, hash_to_send, options)
+      return queue_raw(name, hash_to_send, options)
     end
 
-    def queue_raw(class_name, data={}, options={})
+    def queue_raw(name, data={}, options={})
       params = nil
       hash_to_send = data
-      hash_to_send["class_name"] = class_name unless hash_to_send["class_name"]
-      hash_to_send["name"] = class_name unless hash_to_send["name"]
+      #hash_to_send["class_name"] = name unless hash_to_send["class_name"]
+      hash_to_send["name"] = name unless hash_to_send["name"]
       uri = project_url_prefix(get_project_id(options)) + "tasks"
       SimpleWorker.logger.debug 'queue_raw , uri = ' + uri
       ret = post(uri, hash_to_send)
@@ -460,19 +457,20 @@ end
     #     - end_at:        Scheduled task will stop running after this date (optional, if ommitted, runs forever or until cancelled)
     #     - run_times:     Task will run exactly :run_times. For instance if :run_times is 5, then the task will run 5 times.
     #
-    def schedule(class_name, data, schedule)
-      puts "Scheduling #{class_name}..."
+    def schedule(name, data, schedule)
+      puts "Scheduling #{name}..."
       raise "Schedule must be a hash." if !schedule.is_a? Hash
-#            if !data.is_a?(Array)
-#                data = [data]
-#            end
       hash_to_send = {}
-      hash_to_send["payload"] = data
-      hash_to_send["class_name"] = class_name
-      hash_to_send["schedule"] = schedule
+      schedules = []
+      schedule["payload"] = data.to_json
+      schedule["name"] = name unless schedule["name"]
+      schedule["code_name"] = name unless schedule["code_name"]
+      schedules << schedule
+      hash_to_send["schedules"] = schedules
       add_sw_params(hash_to_send)
 #            puts ' about to send ' + hash_to_send.inspect
-      ret = post("scheduler/schedule", hash_to_send)
+      uri = project_url_prefix(get_project_id(data)) + "schedules"
+      ret = post(uri, hash_to_send)
       ret
     end
 
@@ -542,9 +540,9 @@ end
       ret
     end
 
-    def schedule_status(schedule_id)
+    def schedule_status(schedule_id, options={})
       data = {"schedule_id"=>schedule_id}
-      ret = get("scheduler/status", data)
+      ret = get("#{project_url_prefix(get_project_id(options))}schedules/#{schedule_id}", data)
       ret
     end
 

metadata

@@ -2,7 +2,7 @@
 name: simple_worker
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 2.0.0.beta.11
+  version: 2.0.0.beta.15
 platform: ruby
 authors: 
 - Travis Reeder
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-15 00:00:00 Z
+date: 2011-10-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: zip
@@ -34,6 +34,17 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: patron
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
 description: The official SimpleWorker gem for http://www.simpleworker.com
 email: travis@appoxy.com
 executables: []