sp-job 0.1.17

data/lib/sp/job/common.rb

@@ -0,0 +1,295 @@
+#
+# Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# And this is the mix-in we'll apply to Job execution classes
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+module SP
+  module Job
+    module Common
+
+      def logger
+        Backburner.configuration.logger
+      end
+
+      def id_to_path (id)
+        "%03d/%03d/%03d/%03d" % [
+          (id % 1000000000000) / 1000000000,
+          (id % 1000000000)    / 1000000   ,
+          (id % 1000000)       / 1000      ,
+          (id % 1000)
+        ]
+      end
+
+      def submit_job (args)
+        job      = args[:job]
+        tube     = args[:tube] || $args[:program_name]
+        raise 'missing job argument' unless args[:job]
+
+        validity = args[:validity] || 180
+        ttr      = args[:ttr]      || 60
+        job[:id] = ($redis.incr "#{$config[:service_id]}:jobs:sequential_id").to_s
+        job[:tube] = tube
+        job[:validity] = validity
+        redis_key = "#{$config[:service_id]}:jobs:#{tube}:#{job[:id]}"
+        $redis.pipelined do
+          $redis.hset(redis_key, 'status', '{"status":"queued"}')
+          $redis.expire(redis_key, validity)
+        end
+        $beaneater.tubes[tube].put job.to_json, ttr: ttr
+      end
+
+      def before_perform_init (job)
+
+        if $connected == false
+          database_connect
+          $redis.get "#{$config}:jobs:sequential_id"
+          $connected = true
+        end
+
+        $job_status = {
+          action:       'response',
+          content_type: 'application/json',
+          progress:      0
+        }
+        $report_time_stamp     = 0
+        $job_status[:progress] = 0
+        $exception_reported    = false
+        $publish_key           = $config[:service_id] + ':' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
+        $job_key               = $config[:service_id] + ':jobs:' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
+        $validity              = job[:validity].nil? ? 300 : job[:validity].to_i
+        if $config[:options] && $config[:options][:jsonapi] == true
+          raise "Job didn't specify the mandatory field prefix!" if job[:prefix].blank?
+          $jsonapi.set_url(job[:prefix])
+          init_params = {}
+          init_params[:user_id] = job[:user_id] unless job[:user_id].blank?
+          init_params[:company_id] = job[:company_id] unless job[:company_id].blank?
+          init_params[:company_schema] = job[:company_schema] unless job[:company_schema].blank?
+          init_params[:sharded_schema] = job[:sharded_schema] unless job[:sharded_schema].blank?
+          init_params[:accounting_prefix] = job[:accounting_prefix] unless job[:accounting_prefix].blank?
+          init_params[:accounting_schema] = job[:accounting_schema] unless job[:accounting_schema].blank?
+
+          $jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::Parameters.new(init_params))
+        end
+
+        # Make sure the job is still allowed to run by checking if the key exists in redis
+        unless $redis.exists($job_key )
+          logger.warn "Job validity has expired: job ignored"
+          return false
+        end
+        return true
+      end
+
+      #
+      # Optionally after the jobs runs sucessfully clean the "job" key in redis
+      # 
+      def after_perform_cleanup (job)
+        if false # TODO check key namings with americo $job key and redis key
+          return if $redis.nil?
+          return if $job_key.nil?
+          $redis.del $job_key
+        end
+      end
+
+      def update_progress (args)
+        step     = args[:step]
+        status   = args[:status]
+        progress = args[:progress]
+        barrier  = args[:barrier]
+        p_index  = args[:index]
+        response = args[:response]
+
+        if args.has_key? :message
+          message_args = Hash.new
+          args.each do |key, value|
+            next if [:step, :progress, :message, :status, :barrier, :index, :response].include? key
+            message_args[key] = value
+          end
+          message = [ args[:message], message_args ]
+        else
+          message = nil
+        end
+        $job_status[:progress] = progress.to_f.round(2) unless progress.nil?
+        $job_status[:progress] = ($job_status[:progress] + step.to_f).round(2) unless step.nil?
+        $job_status[:message]  = message unless message.nil?
+        $job_status[:index]    = p_index unless p_index.nil?
+        $job_status[:status]   = status.nil? ? 'in-progress' : status
+        $job_status[:response] = response unless response.nil?
+        if args.has_key? :link
+          $job_status[:link] = args[:link]
+        end
+
+        if args.has_key? :extra
+          args[:extra].each do |key, value|
+            $job_status[key] = value
+          end
+        end
+
+        if status == 'completed' || status == 'error' || (Time.now.to_f - $report_time_stamp) > $min_progress || barrier
+          update_progress_on_redis
+        end
+      end
+
+      def raise_error (args)
+        args[:status] = 'error'
+        update_progress(args)
+        $exception_reported = true
+        raise args[:message]
+      end
+
+      def update_progress_on_redis
+        $redis.pipelined do
+          redis_str = $job_status.to_json
+          $redis.publish $publish_key, redis_str
+          $redis.hset    $job_key, 'status', redis_str
+          $redis.expire  $job_key, $validity
+        end
+        $report_time_stamp = Time.now.to_f
+      end
+
+      def get_jsonapi!(path, params)
+        check_db_life_span()
+        $jsonapi.adapter.get!(path, params)
+      end
+
+      def post_jsonapi!(path, params)
+        check_db_life_span()
+        $jsonapi.adapter.post!(path, params)
+      end
+
+      def patch_jsonapi!(path, params)
+        check_db_life_span()
+        $jsonapi.adapter.patch!(path, params)
+      end
+
+      def delete_jsonapi!(path)
+        check_db_life_span()
+        $jsonapi.adapter.delete!(path)
+      end
+
+      def db_exec (query)
+        $pg.query(query: query)
+      end
+
+      def expand_mail_body (template)
+        if File.extname(template) == ''
+          template += '.erb'
+        end
+        if template[0] == '/'
+          erb_template = File.read(template)
+        else
+          erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)), template))
+        end
+        ERB.new(erb_template).result(binding)
+      end
+
+      def send_email (args)
+        if args.has_key?(:template)
+          email_body = expand_mail_body args[:template]
+        else
+          email_body = args[:body]
+        end
+
+        document = Roadie::Document.new email_body
+        email_body = document.transform
+
+        m = Mail.new do
+          from     $config[:mail][:from]
+          to       args[:to]
+          subject  args[:subject]
+
+          html_part do
+            content_type 'text/html; charset=UTF-8'
+            body email_body
+          end
+        end
+
+        begin
+          m.deliver!
+          # ap m.to_s
+          return OpenStruct.new(status: true)
+        rescue Net::OpenTimeout => e
+          ap ["OpenTimeout", e]
+          return OpenStruct.new(status: false, message: e.message)
+        rescue Exception => e
+          ap e
+          return OpenStruct.new(status: false, message: e.message)
+        end
+
+      end
+
+      def database_connect
+        # any connection to close?
+        if ! $jsonapi.nil?
+          $jsonapi.close
+          $jsonapi = nil
+        end
+        if nil != $pg
+          $pg.disconnect()
+          $pg = nil
+        end
+        # establish new connection?
+        if $config[:postgres] && $config[:postgres][:conn_str]
+          $pg = ::SP::Job::PGConnection.new(owner: 'back_burner', config: $config[:postgres])
+          $pg.connect()
+          if $config[:options][:jsonapi] == true
+            $jsonapi = SP::Duh::JSONAPI::Service.new($pg.connection, ($jsonapi.nil? ? nil : $jsonapi.url))
+          end
+        end
+      end
+
+      def define_db_life_span_treshhold
+        min = $config[:postgres][:min_queries_per_conn]
+        max = $config[:postgres][:max_queries_per_conn]
+        if (!max.nil? && max > 0) || (!min.nil? && min > 0)
+          $db_life_span       = 0
+          $check_db_life_span = true
+          new_min, new_max = [min, max].minmax
+          new_min = new_min if new_min <= 0
+          if new_min + new_min > 0
+            $db_treshold = (new_min + (new_min - new_min) * rand).to_i
+          else
+            $db_treshold = new_min.to_i
+          end
+        end
+      end
+
+      def check_db_life_span
+        return unless $check_db_life_span
+        $db_life_span += 1
+        if $db_life_span > $db_treshold
+          # Reset pg connection
+          database_connect()
+        end
+      end
+
+      def catch_fatal_exceptions (e)
+        case e
+        when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
+          logger.fatal "Lost connection to database exiting now"
+          exit
+        when Redis::CannotConnectError
+          logger.fatal "Can't connect to redis exiting now"
+          exit
+        end
+      end
+
+    end # Module Common
+  end # Module Job
+end # Module SP

data/lib/sp/job/engine.rb

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+
+module SP
+  module Job
+    class Engine < ::Rails::Engine
+      isolate_namespace SP::Job
+
+      initializer :append_migrations do |app|
+        unless app.root.to_s.match root.to_s
+          app.config.paths["db/migrate"] += config.paths["db/migrate"].expanded
+        end
+      end
+    end
+  end
+end

data/lib/sp/job/jsonapi_error.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+#
+# encoding: utf-8
+#
+# Copyright (c) 2017 Cloudware S.A. Allrights reserved
+#
+# Helper to obtain tokens to access toconline API's.
+#
+
+module SP
+  module Job
+
+    #
+    #  Helper class to transform an error into json api format.
+    #
+    class JSONAPIError < StandardError
+
+      # {
+      #   "errors": [
+      #     {
+      #       "status": nullptr,
+      #       "code": nullptr,
+      #       "detail": nullptr,
+      #       "meta": {
+      #         "internal-error": nullptr
+      #       }
+      #     }
+      #   ]
+      # }
+
+      private
+
+        @error
+
+      public
+
+      def initialize (code: 500, detail: nil, internal: nil)
+        @errors = [
+          {
+            :code   => code,
+            :detail => detail
+          }
+        ]
+        # 4xx
+        case code
+        when 400
+          @errors[0][:status] = "Bad Request"
+        when 401
+          @errors[0][:status] = "Unauthorized"
+        when 403
+          @errors[0][:status] = "Forbidden"
+        when 404
+          @errors[0][:status] = "Not Found"
+        when 405
+          @errors[0][:status] = "Method Not Allowed"
+        when 406
+          @errors[0][:status] = "Not Acceptable"
+        when 408
+          @errors[0][:status] = "Request Timeout"
+        # 5xx
+        when 501
+          @errors[0][:status] = "Not Implemented"
+        else
+        # other
+          @errors[0][:status] = "Internal Server Error"
+        end
+        @errors[0][:status] = "#{code} #{@errors[0][:status]}"
+        if nil != internal
+          @errors[0][:meta] = { :'internal-error' => internal }
+        end
+      end
+
+      def code ()
+        return @errors[0][:code]
+      end
+
+      def content_type ()
+        "application/vnd.api+json;charset=utf-8"
+      end
+
+      def body ()
+        {
+          :errors => @errors
+        }
+      end
+
+      def content_type_and_body ()
+        [ content_type(), body() ]
+      end
+
+    end
+
+  end # Job module
+end # SP module

data/lib/sp/job/pg_connection.rb

@@ -0,0 +1,179 @@
+#
+# Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+
+require 'pg'
+require 'openssl'
+
+module SP
+  module Job
+
+    class PGConnection
+
+      private
+
+      #
+      # Private Data
+      #
+      @owner      = nil
+      @config     = nil
+      @connection = nil
+      @treshold   = -1
+      @counter    = 0
+      @statements = []
+
+      public
+
+      #
+      # Public Attributes
+      #
+      attr_accessor :connection
+
+      #
+      # Prepare database connection configuration.
+      #
+      # @param owner
+      # @param config
+      #
+      def initialize (owner:, config:)
+        @owner      = owner
+        @config     = config
+        @connection         = nil
+        @treshold   = -1
+        @counter    = 0
+        @statements = []
+        min = @config[:min_queries_per_conn]
+        max = @config[:max_queries_per_conn]
+        if (!max.nil? && max > 0) || (!min.nil? && min > 0)
+          @counter = 0
+          new_min, new_max = [min, max].minmax
+          new_min = new_min if new_min <= 0
+          if new_min + new_min > 0
+            @treshold = (new_min + (new_min - new_min) * rand).to_i
+          else
+            @treshold = new_min.to_i
+          end
+        end
+      end
+
+      #
+      # Establish a new database connection.
+      #        Previous one ( if any ) will be closed first.
+      #
+      def connect ()
+        disconnect()
+        @connection = PG.connect(@config[:conn_str])
+      end
+
+      #
+      # Close currenly open database connection.
+      #
+      def disconnect ()
+        if @connection.nil?
+          return
+        end
+        while @statements.count > 0 do
+            @connection.exec("DEALLOCATE #{@statements.pop()}")
+        end
+        @connection.close
+        @connection = nil
+        @counter = 0
+      end
+
+      #
+      # Prepare an SQL statement.
+      #
+      # @param query
+      #
+      # @return Statement id.
+      #
+      def prepare_statement (query:)
+        if nil == @connection
+          connect()
+        end
+        id = "#{@owner}_#{Digest::MD5.hexdigest(query)}"
+        if @statements.include? id
+          return id
+        else
+          @statements << id
+          @connection.prepare(@statements.last, query)
+          return @statements.last
+        end
+      end
+
+      #
+      # Execute a previously prepared SQL statement.
+      #
+      # @param id
+      # @param args
+      #
+      # @return PG result
+      #
+      def execute_statement (id:, args:)
+        check_life_span()
+        @connection.exec_prepared(id, args)
+      end
+
+      #
+      # Destroy a previously prepared SQL statement.
+      #
+      # @param id
+      # @param args
+      #
+      def dealloc_statement (id:)
+        if nil == id
+          while @statements.count > 0 do
+              @connection.exec("DEALLOCATE #{@statements.pop()}")
+          end
+        else
+          @statements.delete!(id)
+          @connection.exec("DEALLOCATE #{id}")
+        end
+      end
+
+      #
+      # Execute a query,
+      #
+      # @param query
+      #
+      def query (query:)
+        unless query.nil?
+          check_life_span()
+          @connection.exec(query)
+        end
+      end
+
+      private
+
+      #
+      # Check connection life span
+      #
+      def check_life_span ()
+        return unless @treshold > 0
+        @counter += 1
+        if @counter > @treshold
+          connect()
+        end
+      end
+
+    end # end class 'PGConnection'
+
+  end # module 'Job'
+end # module 'SP'