sp-job 0.1.17 → 0.2.2

data/lib/sp/job/broker.rb

@@ -19,13 +19,6 @@ module SP
       #
       class I18N
 
-        private
-
-        @key
-        @args
-
-        public
-
         attr_accessor :key
         attr_accessor :args
 
@@ -98,14 +91,6 @@ module SP
       #
       class OAuth2
 
-        private
-
-        @service_id = nil
-        @client     = nil
-        @redis      = nil
-
-        public
-
         def initialize (service_id:, config:, redis: nil)
           @service_id = service_id
           @client = ::SP::Job::BrokerOAuth2Client.new(
@@ -131,7 +116,7 @@ module SP
           ac_response = @client.get_authorization_code(
             a_redirect_uri = nil,
             a_scope = scope
-          )          
+          )
           # got a valid 'authorization code'?
           if ac_response[:oauth2].has_key?(:code)
             # got fields?
@@ -142,7 +127,7 @@ module SP
                 array << k.to_s
                 array << v
               end
-              $redis.hmset("#{@service_id}:oauth:authorization_code:#{ac_response[:oauth2][:code]}",
+              @redis.hmset("#{@service_id}:oauth:authorization_code:#{ac_response[:oauth2][:code]}",
                 array,
                 'patched_by', 'toconline-session'
               )
@@ -212,7 +197,7 @@ module SP
             a_scope = nil # keep current scope
           )
           if at_response[:oauth2].has_key?(:error)
-            return at_response
+            raise ::SP::Job::Broker::InternalError.new(i18n: nil, internal: at_response[:oauth2][:error])
           end
           # prepare redis arguments: field value, [field value, ...]
           array = []
@@ -294,6 +279,58 @@ module SP
           }
         end
 
+        #
+        # Obtain an 'access' and a 'refresh' token.
+        #
+        # @param args check the authorize method o OAuth2 class 
+        # @return hash with response content type and status code
+        #
+        def authorize (args)
+          call do
+            finalized(response: oauth2.authorize(args))
+          end
+          @output
+        end
+
+        #
+        # Refresh an access token.
+        #
+        # @param args check the refresh method o OAuth2 class 
+        # @return hash with response content type and status code
+        #
+        def refresh (args)
+          call do
+            finalized(response: oauth2.refresh(args))
+          end
+          @output
+        end
+
+        #
+        # Patch a pair of tokens, by generating new ones
+        #
+        # @param args check the patch methods o OAuth2 class 
+        # @return hash with response content type and status code
+        #
+        def patch (args)
+          call do
+            finalized(response: oauth2.patch(args))
+          end
+          @output
+        end
+
+        #
+        # Remove a pair of tokens from redis.
+        #
+        # @param args check the dispose method o OAuth2 class 
+        # @return hash with response content type and status code
+        #
+        def dispose (args)
+          call do 
+            finalized(response: oauth2.dispose(args))
+          end
+          @output
+        end
+
         #
         # Finalize the job response.
         #
@@ -301,7 +338,7 @@ module SP
         # @param content_type
         # @param status_code
         #
-        # @return
+        # @return hash with response content type and status code
         #
         def finalized (response:, content_type: 'application/json', status_code: 200)
           @output[:response]     = response
@@ -312,11 +349,12 @@ module SP
 
         #
         # Perform an OAuth2 request, catch errors
-        # and convertem them to a common result hash.
+        # and convert them to a common result hash.
         #
         # @param callback
         #
         def call(*callback)
+          @output = {}
           begin
             @output = yield
           rescue ::SP::Job::BrokerOAuth2Client::InvalidEmailOrPassword => invalid_password
@@ -357,7 +395,7 @@ module SP
             @output[:status_code] = broker_error.code
             @output[:content_type], @output[:response] = broker_error.content_type_and_body()
           rescue Exception => e
-            internal_error = InternalError.new(i18n: nil, internal: nil)
+            internal_error = InternalError.new(i18n: nil, internal: e.message)
             @output[:status_code] = internal_error.code
             @output[:content_type], @output[:response] = internal_error.content_type_and_body()
           end

data/lib/sp/job/common.rb

@@ -24,6 +24,84 @@ module SP
   module Job
     module Common
 
+      ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+      def thread_data
+        $thread_data[Thread.current]
+      end
+
+      def http (oauth_client_id:, oauth_client_secret:)
+
+        $http_oauth_clients ||= {}
+        $http_oauth_clients[oauth_client_id] ||= ::SP::Job::BrokerHTTPClient.new(
+                  a_session =  ::SP::Job::BrokerHTTPClient::Session.new(
+                                  a_access_token  = nil,
+                                  a_refresh_token = nil,
+                                  a_scope         = $config[:api][:oauth][:scope]
+                                ),
+                  a_oauth2_client = ::SP::Job::BrokerOAuth2Client.new(
+                    protocol:      $config[:api][:oauth][:protocol],
+                    host:          $config[:api][:oauth][:host],
+                    port:          $config[:api][:oauth][:port],
+                    client_id:     oauth_client_id,
+                    client_secret: oauth_client_secret,
+                    redirect_uri:  $config[:api][:oauth][:redirect_uri],
+                    scope:         $config[:api][:oauth][:scope],
+                    options:       {}
+                  ),
+                  a_refreshed_callback = method(:refreshed_callback),
+                  a_auto_renew_refresh_token = true
+        )
+        $http_oauth_clients[oauth_client_id]
+      end
+
+      #
+      # Called by BrokerHTTPClient when a new session was created
+      # or an older one was refreshed.
+      #
+      def refreshed_callback(a_session)
+        logger.task('#', "Session #{a_session.is_new ? 'created' : 'refreshed' }, access_token=#{a_session.access_token}, refresh_token=#{a_session.refresh_token}")
+      end
+
+      def db
+        $pg
+      end
+
+      def redis
+        $redis
+      end
+
+      def config
+        $config
+      end
+
+      #
+      # Returns the object you should use to perform JSON api requests
+      #
+      # jsonapi.get! (resource, params)
+      # jsonapi.post! (resource, params)
+      # jsonapi.patch! (resource, params)
+      # jsonapi.delete! (resource)
+      #
+      def jsonapi
+        thread_data.jsonapi.adapter
+      end
+
+      #
+      # You should not use this method ... unless ... you REALLY need to overide the JSON:API
+      # parameters defined by the JOB object
+      #
+      def set_jsonapi_parameters (params)
+        thread_data.jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::ParametersNotPicky.new(params))
+      end
+
+      # You should not use this method ... unless ... you REALLY need to overide the JSON:API
+      # parameters defined by the JOB object
+      #
+      def get_jsonapi_parameters
+        HashWithIndifferentAccess.new(JSON.parse(thread_data.jsonapi.parameters.to_json))
+      end
+
       def logger
         Backburner.configuration.logger
       end
@@ -37,7 +115,42 @@ module SP
         ]
       end
 
+      def get_random_folder
+        ALPHABET[rand(26)] + ALPHABET[rand(26)]
+      end
+
+      def send_to_upload_server (src_file:, id:, extension:, entity: 'company')
+        folder = get_random_folder
+        remote_path = File.join(entity, id_to_path(id.to_i), folder)
+        if config[:uploads][:local] == true
+          destination_file = ::SP::Job::Unique::File.create(File.join(config[:uploads][:path], remote_path), extension)
+          FileUtils.cp(src_file, destination_file)
+        else
+          uploads_server = config[:uploads][:server]
+          destination_file = %x[ssh #{uploads_server} unique-file -p #{File.join(config[:uploads][:path], remote_path)} -e #{extension}].strip
+          if $?.exit_status == 0
+            %x[scp #{src_file} #{uploads_server}:#{remote_file}]
+            raise_error(message: 'i18n_upload_to_server_failed') if $?.exit_status != 0
+          else
+            raise_error(message: 'i18n_upload_to_server_failed')
+          end
+        end
+
+        return entity[0] + folder + destination_file[-(6+extension.length)..-1]
+      end
+
       def submit_job (args)
+        if $redis_mutex.nil?
+          rv = _submit_job(args)
+        else
+          $redis_mutex.synchronize {
+            rv = _submit_job(args)
+          }
+        end
+        rv
+      end
+
+      def _submit_job (args)
         job      = args[:job]
         tube     = args[:tube] || $args[:program_name]
         raise 'missing job argument' unless args[:job]
@@ -53,157 +166,215 @@ module SP
           $redis.expire(redis_key, validity)
         end
         $beaneater.tubes[tube].put job.to_json, ttr: ttr
+        "#{tube}:#{job[:id]}"
       end
 
-      def before_perform_init (job)
+      def prepare_job (job)
+        logger.debug "Preparing job id #{job[:id]}".green
 
-        if $connected == false
-          database_connect
-          $redis.get "#{$config}:jobs:sequential_id"
-          $connected = true
-        end
-
-        $job_status = {
-          action:       'response',
+        td = thread_data
+        td.current_job = job
+        td.job_status = {
           content_type: 'application/json',
-          progress:      0
+          progress: [
+            {
+              message: nil,
+              value: 0
+            }
+          ]
         }
-        $report_time_stamp     = 0
-        $job_status[:progress] = 0
-        $exception_reported    = false
-        $publish_key           = $config[:service_id] + ':' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
-        $job_key               = $config[:service_id] + ':jobs:' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
-        $validity              = job[:validity].nil? ? 300 : job[:validity].to_i
+        td.report_time_stamp    = 0
+        td.exception_reported   = false
+        td.job_id               = job[:id]
+        td.publish_key          = $config[:service_id] + ':' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
+        td.job_key              = $config[:service_id] + ':jobs:' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
         if $config[:options] && $config[:options][:jsonapi] == true
           raise "Job didn't specify the mandatory field prefix!" if job[:prefix].blank?
-          $jsonapi.set_url(job[:prefix])
-          init_params = {}
-          init_params[:user_id] = job[:user_id] unless job[:user_id].blank?
-          init_params[:company_id] = job[:company_id] unless job[:company_id].blank?
-          init_params[:company_schema] = job[:company_schema] unless job[:company_schema].blank?
-          init_params[:sharded_schema] = job[:sharded_schema] unless job[:sharded_schema].blank?
-          init_params[:accounting_prefix] = job[:accounting_prefix] unless job[:accounting_prefix].blank?
-          init_params[:accounting_schema] = job[:accounting_schema] unless job[:accounting_schema].blank?
-
-          $jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::Parameters.new(init_params))
+          td.jsonapi.set_url(job[:prefix])
+          td.set_jsonapi_parameters(SP::Duh::JSONAPI::ParametersNotPicky.new(job))
         end
 
         # Make sure the job is still allowed to run by checking if the key exists in redis
-        unless $redis.exists($job_key )
-          logger.warn "Job validity has expired: job ignored"
+        unless $redis.exists(td.job_key )
+          logger.warn "Job validity has expired: job ignored".yellow
           return false
         end
         return true
       end
 
-      #
-      # Optionally after the jobs runs sucessfully clean the "job" key in redis
-      # 
-      def after_perform_cleanup (job)
-        if false # TODO check key namings with americo $job key and redis key
-          return if $redis.nil?
-          return if $job_key.nil?
-          $redis.del $job_key
-        end
-      end
-
       def update_progress (args)
-        step     = args[:step]
+        td = thread_data
         status   = args[:status]
         progress = args[:progress]
-        barrier  = args[:barrier]
-        p_index  = args[:index]
-        response = args[:response]
+        p_index  = args[:index] || 0
 
         if args.has_key? :message
           message_args = Hash.new
           args.each do |key, value|
-            next if [:step, :progress, :message, :status, :barrier, :index, :response].include? key
+            next if [:step, :progress, :message, :status, :barrier, :index, :response, :action, :content_type, :status_code, :link].include? key
             message_args[key] = value
           end
           message = [ args[:message], message_args ]
         else
           message = nil
         end
-        $job_status[:progress] = progress.to_f.round(2) unless progress.nil?
-        $job_status[:progress] = ($job_status[:progress] + step.to_f).round(2) unless step.nil?
-        $job_status[:message]  = message unless message.nil?
-        $job_status[:index]    = p_index unless p_index.nil?
-        $job_status[:status]   = status.nil? ? 'in-progress' : status
-        $job_status[:response] = response unless response.nil?
-        if args.has_key? :link
-          $job_status[:link] = args[:link]
-        end
 
-        if args.has_key? :extra
-          args[:extra].each do |key, value|
-            $job_status[key] = value
+        # update job status
+        if p_index >= td.job_status[:progress].size
+          (1 + p_index - td.job_status[:progress].size).times do
+            td.job_status[:progress] << { message: nil, value: 0 }
           end
         end
+        unless message.nil?
+          td.job_status[:progress][p_index][:message] = message
+        end
+        unless progress.nil?
+          td.job_status[:progress][p_index][:value] = progress.to_f.round(2)
+        end
+        td.job_status[:status]      = status.nil? ? 'in-progress' : status
+        td.job_status[:link]        = args[:link] if args[:link]
+        td.job_status[:status_code] = args[:status_code] if args[:status_code]
+        if args.has_key? :response
+          td.job_status[:response]     = args[:response]
+          td.job_status[:content_type] = args[:content_type]
+          td.job_status[:action]       = args[:action]
+        end
+
+        # Create notification that will be published
+        td.job_notification = {}
+        td.job_notification[:progress]    = progress.to_f.round(2) unless progress.nil?
+        td.job_notification[:message]     = message unless message.nil?
+        td.job_notification[:index]       = p_index unless p_index.nil?
+        td.job_notification[:status]      = status.nil? ? 'in-progress' : status
+        td.job_notification[:link]        = args[:link] if args[:link]
+        td.job_notification[:status_code] = args[:status_code] if args[:status_code]
+        if args.has_key? :response
+          td.job_notification[:response]     = args[:response]
+          td.job_notification[:content_type] = args[:content_type]
+          td.job_notification[:action]       = args[:action]
+        end
 
-        if status == 'completed' || status == 'error' || (Time.now.to_f - $report_time_stamp) > $min_progress || barrier
+        if ['completed', 'error', 'follow-up', 'cancelled'].include?(status) || (Time.now.to_f - td.report_time_stamp) > $min_progress || args[:barrier]
           update_progress_on_redis
         end
       end
 
-      def raise_error (args)
-        args[:status] = 'error'
+      def send_response (args)
+        td = thread_data
+        args[:status]       ||= 'completed'
+        args[:action]       ||= 'response'
+        args[:content_type] ||= 'application/json'
+        args[:response]     ||= {}
+        args[:status_code]  ||= 200
         update_progress(args)
-        $exception_reported = true
-        raise args[:message]
-      end
-
-      def update_progress_on_redis
-        $redis.pipelined do
-          redis_str = $job_status.to_json
-          $redis.publish $publish_key, redis_str
-          $redis.hset    $job_key, 'status', redis_str
-          $redis.expire  $job_key, $validity
-        end
-        $report_time_stamp = Time.now.to_f
+        td.job_id = nil
       end
 
-      def get_jsonapi!(path, params)
-        check_db_life_span()
-        $jsonapi.adapter.get!(path, params)
+      def error_handler(args)
+        td = thread_data
+        args[:status]       ||= 'error'
+        args[:action]       ||= 'response'
+        args[:content_type] ||= 'application/json'
+        args[:status_code]  ||= 500
+        update_progress(args)
+        logger.error(args)
+        td.exception_reported = true
+        td.job_id = nil
       end
 
-      def post_jsonapi!(path, params)
-        check_db_life_span()
-        $jsonapi.adapter.post!(path, params)
+      def report_error (args)
+        td = thread_data
+        error_handler(args)
+        raise ::SP::Job::JobAborted.new(args: args, job: td.current_job)
       end
 
-      def patch_jsonapi!(path, params)
-        check_db_life_span()
-        $jsonapi.adapter.patch!(path, params)
+      def raise_error (args)
+        td = thread_data
+        error_handler(args)
+        raise ::SP::Job::JobException.new(args: args, job: td.current_job)
       end
 
-      def delete_jsonapi!(path)
-        check_db_life_span()
-        $jsonapi.adapter.delete!(path)
+      def send_text_response (response)
+        td = thread_data
+        $redis.pipelined do
+          $redis.publish td.publish_key, response
+          $redis.hset    td.job_key, 'status', response
+        end
       end
 
-      def db_exec (query)
-        $pg.query(query: query)
+      def update_progress_on_redis
+        td = thread_data
+        if $redis_mutex.nil?
+          if $transient_job
+            $redis.publish td.publish_key, td.job_notification.to_json
+          else
+            $redis.pipelined do
+              $redis.publish td.publish_key, td.job_notification.to_json
+              $redis.hset    td.job_key, 'status', td.job_status.to_json
+            end
+          end
+        else
+          $redis_mutex.synchronize {
+            if $transient_job
+              $redis.publish td.publish_key, td.job_notification.to_json
+            else
+              $redis.pipelined do
+                $redis.publish td.publish_key, td.job_notification.to_json
+                $redis.hset    td.job_key, 'status', td.job_status.to_json
+              end
+            end
+          }
+        end
+        td.report_time_stamp = Time.now.to_f
       end
 
       def expand_mail_body (template)
-        if File.extname(template) == ''
-          template += '.erb'
+        if template.class == Hash
+          template_path = template[:path]
+          erb_binding   = OpenStruct.new(template[:args]).instance_eval { binding }
+        else
+          template_path = template
+          erb_binding = binding
+        end
+
+        if File.extname(template_path) == ''
+          template_path += '.erb'
         end
-        if template[0] == '/'
-          erb_template = File.read(template)
+
+        if template_path[0] == '/'
+          erb_template = File.read(template_path)
         else
-          erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)), template))
+          erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)), template_path))
         end
-        ERB.new(erb_template).result(binding)
+
+        ERB.new(erb_template).result(erb_binding)
       end
 
       def send_email (args)
-        if args.has_key?(:template)
+
+        if args.has_key?(:body) && args[:body] != nil
+          email_body = args[:body]
+        elsif args.has_key?(:template) && args[:template] != nil
           email_body = expand_mail_body args[:template]
-        else
+        end
+
+        submit_job(
+            tube: 'mail-queue',
+            job: {
+              to:       args[:to],
+              subject:  args[:subject],
+              reply_to: args[:reply_to],
+              body:     email_body
+            }
+          )
+      end
+
+      def synchronous_send_email (args)
+
+        if args.has_key?(:body) && args[:body] != nil
           email_body = args[:body]
+        elsif args.has_key?(:template) && args[:template] != nil
+          email_body = expand_mail_body args[:template]
         end
 
         document = Roadie::Document.new email_body
@@ -213,6 +384,7 @@ module SP
           from     $config[:mail][:from]
           to       args[:to]
           subject  args[:subject]
+          reply_to (args[:reply_to] || $config[:mail][:from])
 
           html_part do
             content_type 'text/html; charset=UTF-8'
@@ -220,76 +392,36 @@ module SP
           end
         end
 
-        begin
-          m.deliver!
-          # ap m.to_s
-          return OpenStruct.new(status: true)
-        rescue Net::OpenTimeout => e
-          ap ["OpenTimeout", e]
-          return OpenStruct.new(status: false, message: e.message)
-        rescue Exception => e
-          ap e
-          return OpenStruct.new(status: false, message: e.message)
-        end
-
-      end
-
-      def database_connect
-        # any connection to close?
-        if ! $jsonapi.nil?
-          $jsonapi.close
-          $jsonapi = nil
-        end
-        if nil != $pg
-          $pg.disconnect()
-          $pg = nil
-        end
-        # establish new connection?
-        if $config[:postgres] && $config[:postgres][:conn_str]
-          $pg = ::SP::Job::PGConnection.new(owner: 'back_burner', config: $config[:postgres])
-          $pg.connect()
-          if $config[:options][:jsonapi] == true
-            $jsonapi = SP::Duh::JSONAPI::Service.new($pg.connection, ($jsonapi.nil? ? nil : $jsonapi.url))
+        if args.has_key?(:attachments) && args[:attachments] != nil
+          args[:attachments].each do |attach|
+            attach_uri = URI.escape("#{attach[:protocol]}://#{attach[:host]}:#{attach[:port]}/#{attach[:path]}/#{attach[:file]}")
+            attach_http_call = Curl::Easy.http_get(attach_uri)
+            if attach_http_call.response_code == 200
+              attributes = {}
+              attach_http_call.header_str.scan(/(\w+)="([^"]*)"/).each do |group|
+                attributes[group[0].to_sym] = group[1]
+              end
+
+              m.attachments[attributes[:filename].force_encoding('UTF-8')] = { mime_type: attach_http_call.content_type, content: attach_http_call.body_str }
+            end
           end
         end
-      end
 
-      def define_db_life_span_treshhold
-        min = $config[:postgres][:min_queries_per_conn]
-        max = $config[:postgres][:max_queries_per_conn]
-        if (!max.nil? && max > 0) || (!min.nil? && min > 0)
-          $db_life_span       = 0
-          $check_db_life_span = true
-          new_min, new_max = [min, max].minmax
-          new_min = new_min if new_min <= 0
-          if new_min + new_min > 0
-            $db_treshold = (new_min + (new_min - new_min) * rand).to_i
-          else
-            $db_treshold = new_min.to_i
-          end
-        end
+        m.deliver!
       end
 
-      def check_db_life_span
-        return unless $check_db_life_span
-        $db_life_span += 1
-        if $db_life_span > $db_treshold
-          # Reset pg connection
-          database_connect()
-        end
-      end
+      def pg_server_error(e)
+        raise e if e.is_a?(::SP::Job::JobCancelled)
+        base_exception = e
+        begin
+          base_exception = base_exception.cause
+        end while base_exception.respond_to?(:cause) && !base_exception.cause.blank?
 
-      def catch_fatal_exceptions (e)
-        case e
-        when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
-          logger.fatal "Lost connection to database exiting now"
-          exit
-        when Redis::CannotConnectError
-          logger.fatal "Can't connect to redis exiting now"
-          exit
-        end
+        return base_exception.is_a?(PG::ServerError) ? e.cause.result.error_field(PG::PG_DIAG_MESSAGE_PRIMARY) : e.message
       end
 
+      def get_percentage(total: 1, count: 0) ; (count * 100 / total).to_i ; end
+
     end # Module Common
   end # Module Job
-end # Module SP
+end # Module SP