sp-job 0.2.3 → 0.3.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9672916da9a019a5b292a332195e307e2cf35b5b
-  data.tar.gz: 48c48bd95897811d2a7c922313bd0ebaa8dd27c2
+  metadata.gz: dd0c49ededce3afbbd5e09db28e113b0d6e66790
+  data.tar.gz: 511e7878308035eb4e69bb7864c9205479cbdba8
 SHA512:
-  metadata.gz: 167ac6ad6e86de69551c4241d27154a56701819cac80cdbcdadf69f66fc1dff1afa63b26bdece1705f06a911bae01a8ec6a5aa0cfa0e557a811b8583e1a87c92
-  data.tar.gz: d9bee2c1dbdd785f42c674f6247255fc218a3f068352318ee25e1de5fab1c93f9addb21bd68b7bc28b26a5bb1518ad8f77fd92be12d8b525c8f6abac342725e4
+  metadata.gz: 2ac3252e84c0994f71c0c1d748080e7d6313378eecabc0f7caa1427cb8bbf77943a563c2bdf09a8242499394022ec4e68b45072dca3d52032553bf3fcf6b7801
+  data.tar.gz: 683adc1dbe186ad0034e1762180332b733cb1fd3233139079a05e89ec6ab0f5f1c9e9d4e342f2a36f528d344f46850b5dcd9ae640f2ce749462c6d67e96f795f

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.3.22

data/bin/configure

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2011-2017 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+require 'bundler/setup'
+require 'optparse'
+require 'sp/job/configure/configure'
+
+$args = {}
+
+#
+# Parse command line arguments
+#
+$option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options]"
+  opts.on('-m', '--dump'      , "dump merged configuration")                               { $args[:print_config] = true        }
+  opts.on('-o', '--overwrite' , "overwrite system, project and user files, BE CAREFULL!")  { $args[:action]       = 'overwrite' }
+  opts.on('-h', '--hotfix'    , "hotfix modifies only project and user files")             { $args[:action]       = 'hotfix'    }
+  opts.on('-d', '--debug'     , "developer mode: log to stdout and print job")             { $args[:debug]        = true        }
+end
+$option_parser.parse!
+
+run_configure($args)
+

data/lib/sp-job.rb

@@ -35,12 +35,31 @@ require 'ostruct'
 require 'json'
 require 'mail'
 require 'uri'
+require 'securerandom'
 
 require 'sp/job'
 require 'sp/job/version'
 require 'sp/job/worker'
 require 'sp/job/worker_thread'
+require 'sp/job/rfc822'
 require 'sp/job/common'
 require 'sp/job/unique_file'
-require 'sp/job/broker_http_client'      unless RUBY_ENGINE == 'jruby'
-require 'sp/job/broker_oauth2_client'    unless RUBY_ENGINE == 'jruby'
+require 'sp/job/jwt'
+require 'sp/job/http_client'
+require 'sp/job/query_params'
+require 'sp/job/broker_http_client'    unless RUBY_ENGINE == 'jruby'
+require 'sp/job/broker_oauth2_client'  unless RUBY_ENGINE == 'jruby'
+
+# JSONAPI library classes
+require 'sp/jsonapi'
+require 'sp/jsonapi/exceptions'
+# Parameters class
+require 'sp/jsonapi/parameters'
+# Service classes
+require 'sp/jsonapi/service'
+# Adpater classes
+require 'sp/jsonapi/adapters/base'
+require 'sp/jsonapi/adapters/raw_db'
+require 'sp/jsonapi/adapters/db'
+
+require 'sp/jsonapi/model/base'

data/lib/sp/job/back_burner.rb

@@ -19,10 +19,80 @@
 # encoding: utf-8
 #
 require 'sp/job/pg_connection'
+require 'sp/job/session'
+require 'sp/job/broker'
 require 'sp/job/job_db_adapter'
+require 'sp/job/jsonapi_error'
+require 'sp/job/broker_oauth2_client' unless RUBY_ENGINE == 'jruby' # ::SP::Job::BrokerOAuth2Client::InvalidToken
 require 'roadie'
 require 'thread'
 
+#
+# Helper class that encapsulates the objects needed to access each cluster
+#
+class ClusterMember
+
+  attr_reader :redis     # connection to cluster redis
+  attr_reader :session   # access to redis session
+  attr_reader :db        # database connection
+  attr_reader :number    # cluster number, 1, 2 ...
+  attr_reader :config    # cluster configuration read from the conf.json
+  attr_reader :broker    # Driver for casper broker OAUTH 2.0 authentication
+
+  #
+  # Create the cluster member wrapper
+  #
+  # @param configuration the cluster member configuration structure
+  # @param serviceId the service prefix used by casper redis keys
+  # @param db a fb connection to reuse or nil if a new one should be created
+  #
+  def initialize (configuration:, serviceId:, db: nil)
+    @redis = Redis.new(:host => configuration[:redis][:casper][:host], :port => configuration[:redis][:casper][:port], :db => 0)
+    @session = ::SP::Job::Session.new(configuration: configuration, serviceId: serviceId, redis: @redis, programName: $args[:program_name])
+    if db.nil?
+      @db = ::SP::Job::PGConnection.new(owner: 'back_burner', config: configuration[:db])
+    else
+      @db = db
+    end
+    @number = configuration[:number]
+    @config = configuration
+    unless RUBY_ENGINE == 'jruby'
+      @broker = ::SP::Job::Broker::Job.new(config: {
+                                            :service_id => serviceId,
+                                            :oauth2 => configuration[:oauth2],
+                                            :redis => @redis
+                                          })
+    end
+  end
+
+  #
+  # Returns the global logger object, borrowed from common.rb
+  #
+  def self.logger
+    Backburner.configuration.logger
+  end
+
+  #
+  # Creates the global structure that contains the cluster configuration
+  #
+  def self.configure_cluster
+    $cluster_members = {}
+
+    $config[:cluster][:members].each do |cfg|
+      next if !cfg[:exclude_member].nil? && cfg[:exclude_member] == true
+      cfg[:db][:conn_str] = pg_conn_str(cfg[:db])
+      if cfg[:number] == $config[:runs_on_cluster]
+        $cluster_members[cfg[:number]] = ClusterMember.new(configuration: cfg, serviceId: $config[:service_id], db: $pg)
+      else
+        $cluster_members[cfg[:number]] = ClusterMember.new(configuration: cfg, serviceId: $config[:service_id])
+      end
+      logger.info "Cluster member #{cfg[:number]}: #{cfg[:url]} db=#{cfg[:db][:host]}:#{cfg[:db][:port]}(#{cfg[:db][:dbname]}) redis=#{cfg[:redis][:casper][:host]}:#{cfg[:redis][:casper][:port]}#{' <=' if cfg[:number] == $config[:runs_on_cluster]}"
+    end
+  end
+
+end
+
+
 module SP
   module Job
 
@@ -39,7 +109,7 @@ module SP
       attr_reader :args
 
       def initialize (args:, job: nil)
-	      super(args[:message] || $current_job[:tube] || $args[:program_name])
+	      super(args[:message] || $args[:program_name])
         @job     = job
         @args    = args
       end
@@ -57,11 +127,15 @@ module SP
       end
     end
 
-    class ThreadData < Struct.new(:job_status, :report_time_stamp, :exception_reported, :job_id, :publish_key, :job_key, :current_job, :job_notification, :jsonapi)
+    class ThreadData < Struct.new(:job_status, :report_time_stamp, :exception_reported, :job_id, :publish_key, :job_key, :current_job, :job_notification, :jsonapi, :job_tube, :notification_lock, :notification_lock_key)
       def initialize
-        @job_status = {}
-        if $config[:options] && $config[:options][:jsonapi] == true
-          @jsonapi = SP::Duh::JSONAPI::Service.new($pg, nil, SP::Job::JobDbAdapter)
+        self.job_status = {}
+        if $config[:jsonapi] && $config[:jsonapi][:prefix]
+          if RUBY_ENGINE == 'jruby'  # TODO suck in the base class from SP-DUH
+            self.jsonapi = SP::JSONAPI::Service.new($pg, $config[:jsonapi][:prefix], SP::Job::JobDbAdapter)
+          else
+            self.jsonapi = SP::Duh::JSONAPI::Service.new($pg, $config[:jsonapi][:prefix], SP::Job::JobDbAdapter)
+          end
         end
       end
     end
@@ -75,6 +149,15 @@ module SP
   end
 end
 
+#
+# Helper to build BG connection strings
+#
+def pg_conn_str (config, app_name = nil)
+  if app_name.nil?
+    app_name = "application_name=#{$args[:program_name]}"
+  end
+  return "host=#{config[:host]} port=#{config[:port]} dbname=#{config[:dbname]} user=#{config[:user]}#{config[:password] && config[:password].size != 0 ? ' password='+ config[:password] : '' } #{app_name}"
+end
 
 #
 # Initialize global data needed for configuration
@@ -82,6 +165,7 @@ end
 $prefix           = OS.mac? ? '/usr/local' : ''
 $rollbar          = false
 $min_progress     = 3
+$gracefull_exit   = false
 $args = {
   stdout:           false,
   log_level:        'info',
@@ -95,11 +179,11 @@ $args = {
 #
 $option_parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} ARGS"
-  opts.on('-c', '--config=CONFIG.JSON', "path to json configuration file (default: '#{$args[:default_log_file]}')") { |v| $args[:config_file] = File.expand_path(v) }
-  opts.on('-l', '--log=LOGFILE'       , "path to log file (default: '#{$args[:log_file]}')")                        { |v| $args[:log_file]    = File.expand_path(v) }
-  opts.on('-d', '--debug'             , "developer mode: log to stdout and print job")                              { $args[:debug]           = true                }
-  opts.on('-v', '--log_level=LEVEL'   , "Log level DEBUG, INFO, WARN, ERROR, FATAL")                                { |v| $args[:log_level]   = v                   }
-  opts.on('-i', '--index=IDX'         , "systemd instance index")                                                   { |v| $args[:index]       = v                   }
+  opts.on('-c', '--config=CONFIG.JSON', "path to json configuration file (default: '#{$args[:config_file]}')") { |v| $args[:config_file] = File.expand_path(v) }
+  opts.on('-l', '--log=LOGFILE'       , "path to log file (default: '#{$args[:default_log_file]}')")           { |v| $args[:log_file]    = File.expand_path(v) }
+  opts.on('-d', '--debug'             , "developer mode: log to stdout and print job")                         { $args[:debug]           = true                }
+  opts.on('-v', '--log_level=LEVEL'   , "Log level DEBUG, INFO, WARN, ERROR, FATAL")                           { |v| $args[:log_level]   = v                   }
+  opts.on('-i', '--index=IDX'         , "systemd instance index")                                              { |v| $args[:index]       = v                   }
 end
 $option_parser.parse!
 
@@ -131,13 +215,6 @@ File.write("#{$prefix}/var/run/jobs/#{$args[:program_name]}#{$args[:index].nil?
 $config = JSON.parse(File.read(File.expand_path($args[:config_file])), symbolize_names: true)
 $min_progress = $config[:options][:min_progress]
 
-#
-# Global data for mutex and sync
-#
-$threads = [ Thread.current ]
-$thread_data = {}
-$thread_data[Thread.current] = ::SP::Job::ThreadData.new
-
 #
 # Sanity check we only support multithreading on JRUBY
 #
@@ -156,7 +233,13 @@ end
 # Configure rollbar
 #
 unless $config[:rollbar].nil?
-  $rollbar = true
+
+  if $config[:rollbar][:enabled] == 'false'
+    $rollbar = false
+  else
+    $rollbar = true
+  end
+
   Rollbar.configure do |config|
     config.access_token = $config[:rollbar][:token] if $config[:rollbar][:token]
     config.environment  = $config[:rollbar][:environment] if $config[:rollbar] && $config[:rollbar][:environment]
@@ -166,36 +249,132 @@ end
 #
 # Configure backburner queue
 #
+
+class InternalBrokerException
+
+  def self.handle(task:, exception:, hooks:, callback:)
+
+    response = InternalBrokerException.translate_to_response(e: exception)
+    job_id   = task.id
+    job_body = JSON.parse(task.body, symbolize_names: true)
+
+    job_options = hooks[:var].invoke_hook_events(hooks[:klass], :on_raise_response_was_sent, job_id, job_body, response)
+    if nil == job_options || false == job_options.is_a?(Array) || 1 != job_options.size || false == job_options[0].is_a?(Hash)
+      rv = { response: response }
+    else
+      job_options = job_options[0]
+      # rollbar it
+      if true == job_options.include?(:rollbar) && true == job_options[:rollbar]
+        InternalBrokerException.rollbar(job_id: job_id, job_body: job_body, exception: exception)
+      end
+      #
+      rv = { bury: job_options[:bury] , raise: job_options[:raise], response: job_options[:response] || response }
+    end
+
+    # send response
+    callback.call(rv.delete(:response))
+
+    # done
+    rv
+
+  end
+
+  def self.translate_to_response(e:)
+    args = {}
+    args[:status] = 'error'
+    args[:action] = 'response'
+    if e.is_a?(::SP::Job::JSONAPI::Error)
+      args[:status_code]  = e.status_code
+      args[:content_type] = e.content_type
+      args[:response]     = e.body
+    elsif e.is_a?(::SP::Job::BrokerOAuth2Client::InvalidToken)
+      args[:status_code]  = 401
+      args[:content_type] = ''
+      args[:response]     = ''
+    else
+      e = ::SP::Job::JSONAPI::Error.new(status: 500, code: '999', detail: e.message)
+      args[:status_code]  = e.status_code
+      args[:content_type] = e.content_type
+      args[:response]     = e.body
+    end
+    args
+  end
+
+  #
+  # Report exception to rollbar
+  #
+  # @param j Job ID
+  # @param a Job Body
+  # @param e Exception
+  #
+  def self.rollbar(job_id:, job_body:, exception:)
+    $roolbar_mutex.synchronize {
+      if $rollbar
+        if exception.instance_of? ::SP::Job::JobException
+          exception.job[:password] = '<redacted>'
+          Rollbar.error(exception, exception.message, { job: exception.job, args: exception.args})
+        elsif exception.is_a?(::SP::Job::JSONAPI::Error)
+          [:access_token, :refresh_token, :password].each do | s |
+            if job_body.has_key?(s)
+              job_body[s] = '<redacted>'
+            end
+          end
+          Rollbar.error(exception, exception.message, { job: job_id, args: job_body, response: exception.body })
+        else
+          Rollbar.error(exception)
+        end
+      end
+    }
+  end
+
+end
+
 Backburner.configure do |config|
 
   config.beanstalk_url = "beanstalk://#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
   config.on_error      = lambda { |e|
     td = thread_data
+
+    # ensure currently open ( if any ) transaction rollback
+    $pg.rollback unless ! $pg
+
     if td.exception_reported == false
       td.exception_reported = true
       if e.instance_of? Beaneater::DeadlineSoonError
         logger.warn "got a deadline warning".red
       else
         begin
-          raise_error(message: e)
-        rescue => e
+          if $config[:options] && $config[:options][:source] == 'broker'
+            send_response(InternalBrokerException.translate_to_response(e:e))
+          else
+            if e.is_a?(::SP::Job::JobAborted) || e.is_a?(::SP::Job::JobException)
+              raise_error(message: e)
+            else
+              raise_error(message: 'i18n_unexpected_server_error')
+            end
+          end
+        rescue
           # Do not retrow!!!!
         end
       end
     end
-
     # Report exception to rollbar
     $roolbar_mutex.synchronize {
       if $rollbar
         if e.instance_of? ::SP::Job::JobException
           e.job[:password] = '<redacted>'
           Rollbar.error(e, e.message, { job: e.job, args: e.args})
+        elsif e.is_a?(::SP::Job::JSONAPI::Error)
+          Rollbar.error(e, e.body)
         else
           Rollbar.error(e)
         end
       end
     }
 
+    # Signal job termination
+    td.job_id = nil
+
     # Catch fatal exception that must be handled with a restarts (systemctl will restart us)
     case e
     when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
@@ -307,34 +486,31 @@ module Backburner
 
   module Logger
 
-    if RUBY_ENGINE != 'jruby'
-
-      def log_job_begin(name, args)
-        log_info "Work job #{name}"
-        @job_started_at = Time.now
-      end
-
-    else
-
-      def log_job_begin(name, args)
-        log_info "Work job #{name}"
-        Thread.current[:job_started_at] = Time.now
-      end
-
-      # Print out when a job completed
-      # If message is nil, job is considered complete
-      def log_job_end(name, message = nil)
-        ellapsed = Time.now - Thread.current[:job_started_at]
-        ms = (ellapsed.to_f * 1000).to_i
-        action_word = message ? 'Finished' : 'Completed'
-        log_info("#{action_word} #{name} in #{ms}ms #{message}")
+    def log_job_begin(name, args)
+      param_log = ''
+      args = args[0]
+      [ :user_id, :entity_id, :entity_schema, :sharded_schema, :subentity_id, :subentity_prefix, :subentity_schema, :action].each do |key|
+        if args.has_key?(key) && !(args[key].nil? || args[key].empty?)
+          param_log += "#{key}: #{args[key]},"
+        end
       end
+      log_info "Job ##{args[:id]} started #{name}: #{param_log}".white
+      Thread.current[:job_started_at] = Time.now
+    end
 
+    # Print out when a job completed
+    # If message is nil, job is considered complete
+    def log_job_end(name, message = nil)
+      ellapsed = Time.now - Thread.current[:job_started_at]
+      ms = (ellapsed.to_f * 1000).to_i
+      action_word = message ? 'finished' : 'completed'
+      log_info "Job ##{$thread_data[Thread.current][:current_job][:id]} #{action_word} (#{name}) in #{ms}ms #{message}".white
     end
+
   end
 
   class Job
-    extend SP::Job::Common # to bring in logger and report_error into this class
+    include SP::Job::Common # to bring in logger and report_error into this class
 
     # Processes a job and handles any failure, deleting the job once complete
     #
@@ -342,16 +518,24 @@ module Backburner
     #   @task.process
     #
     def process
+      td = thread_data
+
       # Invoke the job setup function, bailout if the setup returns false
       unless job_class.respond_to?(:prepare_job) && job_class.prepare_job(*args)
-        task.delete
         logger.warn "Delete stale or preempted task".red
+
+        # Signal job termination and remove from queue
+        td.job_id = nil
+        task.delete
         return false
       end
 
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_class, :before_perform, *args)
+      @hooks.invoke_hook_events(job_class, :prepend_platform_configuration, *args)
       unless res
+        # Signal job termination and remove from queue
+        td.job_id = nil
         task.delete
         return false
       end
@@ -362,76 +546,174 @@ module Backburner
         #  b) ttr == 1, so that we don't accidentally set it to never time out
         #  NB: A ttr of 1 will likely result in race conditions between
         #  Backburner and beanstalkd and should probably be avoided
-        timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
+        if task.stats.nil?
+          ttr = 60 # Experimental
+        elsif task.ttr > 1
+          ttr = task.ttr - 1
+        else
+          ttr = task.ttr
+        end
+
+        timeout_job_after(ttr) { job_class.perform(*args) }
       end
       task.delete
       # Invoke after perform hook
       @hooks.invoke_hook_events(job_class, :after_perform, *args)
-    rescue ::SP::Job::JobAborted => ja
+      # ensure currently open ( if any ) transaction rollback
+      $pg.rollback unless ! $pg
+    rescue ::SP::Job::JobAborted
       #
       # This exception:
       #  1. is sent to the rollbar
       #  2. does not bury the job, instead the job is deleted
       #
-      logger.debug "Received job aborted exception #{Thread.current}".yellow
       unless task.nil?
-        logger.debug 'Task deleted'.yellow
         task.delete
       end
       # Invoke after perform hook
       @hooks.invoke_hook_events(job_class, :after_perform, *args)
-      thread_data.job_id = nil
+      td.job_id = nil
+      # ensure currently open ( if any ) transaction rollback
+      $pg.rollback unless ! $pg
     rescue ::SP::Job::JobCancelled => jc
       #
       # This exception:
       #  1. is not sent to the rollbar
       #  2. does not bury the job, instead the job is deleted
       #
-      logger.debug "Received job cancellation exception #{Thread.current}".yellow
+      logger.info "Received job cancellation exception #{Thread.current}".yellow
       unless task.nil?
-        logger.debug 'Task deleted'.yellow
-        task.delete
+        logger.info 'Task deleted'.yellow
+        begin
+          task.delete
+        rescue Beaneater::NotFoundError => bnfe
+          @hooks.invoke_hook_events(job_class, :on_failure, bnfe, *args)
+        end
       end
       @hooks.invoke_hook_events(job_class, :on_failure, jc, *args)
-      report_error(message: 'i18n_job_cancelled', status: 'cancelled')
+      error_handler(message: 'i18n_job_cancelled', status: 'cancelled')
       if $redis_mutex.nil?
-        $redis.hset(thread_data.job_key, 'cancelled', true)
+        $redis.hset(td.job_key, 'cancelled', true)
       else
         $redis_mutex.synchronize {
-          $redis.hset(thread_data.job_key, 'cancelled', true)
+          $redis.hset(td.job_key, 'cancelled', true)
         }
       end
-      thread_data.job_id = nil
+      td.job_id = nil
+      # ensure currently open ( if any ) transaction rollback
+      $pg.rollback unless ! $pg
     rescue => e
-      @hooks.invoke_hook_events(job_class, :on_failure, e, *args)
-      raise e
+      # ensure currently open ( if any ) transaction rollback
+
+      if $pg
+        $pg.rollback
+        if e.is_a?(Backburner::Job::JobTimeout)
+          logger.info 'RESET PG connection because Backburner::Job::JobTimeout could be inside PG'.red
+          $pg.connect()
+        end
+      end
+
+      # if we're in broker mode
+      if $config[:options] && $config[:options][:source] == 'broker'
+        # prepare next action for this exception
+        exception_options = {
+          bury: $config[:options].has_key?(:bury) ? $config[:options][:bury] || false : false,
+          raise: true
+        }
+        begin
+          tmp = InternalBrokerException.handle(task: task, exception: e, hooks: { klass: job_class, var:@hooks }, callback: method(:send_response))
+          exception_options[:bury]  = tmp.has_key?(:bury)  ? tmp[:bury]  : exception_options[:bury]
+          exception_options[:raise] = tmp.has_key?(:raise) ? tmp[:raise] : exception_options[:raise]
+        rescue => ne
+          @hooks.invoke_hook_events(job_class, :on_failure, ne, *args)
+          raise ne
+        end
+        # delete it now?
+        if nil != task
+          if true == exception_options[:bury]
+            task.bury
+          else
+            task.delete
+          end
+        end
+        # re-raise?
+        if true == exception_options[:raise]
+          raise e
+        end
+      else
+        @hooks.invoke_hook_events(job_class, :on_failure, e, *args)
+        raise e
+      end
+      # Signal job termination
+      td.job_id = nil
     end
   end
+
 end
 
 # Mix-in the common mix-in to make code available for the lambdas used in this file
 extend SP::Job::Common
 
-logger.debug "Log file ... #{$args[:log_file]}"
-logger.debug "PID ........ #{Process.pid}"
+logger.info "Log file ...... #{$args[:log_file]}"
+logger.info "PID ........... #{Process.pid}"
 
 #
 # Now create the global data needed by the mix-in methods
 #
 $connected     = false
+
+#### TODO read config direct from cluster that will allow unification of jobs configs
+
+# Cluster age config here
+
+#### TODO else classical config
+
 $redis         = Redis.new(:host => $config[:redis][:host], :port => $config[:redis][:port], :db => 0)
-$transient_job = $config[:options] && $config[:options][:transient] == true
+$transient_job = $config[:options] && ( $config[:options][:transient] == true || $config[:options][:source] == 'broker' )
+# raw_response, in the job conf.json can either be:
+# - a Boolean (true or false)
+# - an Array of tube names; in this case, the response will be raw if the current tube name is one of the Array names
+$raw_response  = ($config[:options] ? ($config[:options][:raw_response].nil? ? false : $config[:options][:raw_response]) : false)
+$verbose_log   = $config[:options] && $config[:options][:verbose_log] == true
 $beaneater     = Beaneater.new "#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
 if $config[:postgres] && $config[:postgres][:conn_str]
-  $pg = ::SP::Job::PGConnection.new(owner: $PROGRAM_NAME, config: $config[:postgres], multithreaded: $multithreading)
-  if $PROGRAM_NAME.split('/').last == 'saft-importer' || $PROGRAM_NAME.split('/').last == 'saft-destroyer'
+  $pg = ::SP::Job::PGConnection.new(owner: $args[:program_name], config: $config[:postgres], multithreaded: $multithreading)
+  if $verbose_log
     $pg.exec("SET log_min_duration_statement TO 0;")
   end
-  if $config[:options][:jsonapi] == true
-    $jsonapi = SP::Duh::JSONAPI::Service.new($pg, ($jsonapi.nil? ? nil : $jsonapi.url), SP::Job::JobDbAdapter)
+end
+
+#### TODO end ####
+
+# Check if the user DB is on a different database
+if config[:cluster]
+  if config[:cluster][:user_db].instance_of? Hash
+    config[:cluster][:user_db][:conn_str] = pg_conn_str(config[:cluster][:user_db])
+    $user_db = ::SP::Job::PGConnection.new(owner: $PROGRAM_NAME, config: config[:cluster][:user_db], multithreaded: $multithreading)
+    logger.info "Central DB .... #{$user_db.config[:host]}:#{$user_db.config[:port]}(#{$user_db.config[:dbname]})"
+  else
+    $user_db = nil # Will be grabbed from $cluster_members
+    logger.info "Central DB .... embedded in cluster #{config[:cluster][:user_db]}"
   end
 end
 
+$excluded_members = $config[:cluster].nil? ? [] : ( $config[:cluster][:members].map {|m| m[:number] if m[:exclude_member] }.compact )
+
+#
+# Global data for mutex and sync
+#
+$threads = [ Thread.current ]
+$thread_data = {}
+$thread_data[Thread.current] = ::SP::Job::ThreadData.new
+
+#
+# Signal handler
+#
+Signal.trap('SIGUSR2') {
+  $gracefull_exit = true
+  check_gracefull_exit(dolog: false)
+}
+
 #
 # Open a second thread that will listen to cancellation and other "signals"
 #
@@ -443,17 +725,17 @@ $cancel_thread = Thread.new {
         begin
           message = JSON.parse(msg, {symbolize_names: true})
           $threads.each do |thread|
-            if $thread_data[thread].job_id != nil && message[:id].to_s == $thread_data[thread].job_id && message[:status] == 'cancelled'
+            if $thread_data[thread].job_id != nil && message[:id].to_s == $thread_data[thread].job_id.to_s && message[:status] == 'cancelled'
               logger.info "Received cancel signal for job #{$thread_data[thread].job_id}"
               thread.raise(::SP::Job::JobCancelled.new)
             end
           end
-        rescue Exception => e
+        rescue Exception
           # ignore invalid payloads
         end
       end
     end
-  rescue Redis::CannotConnectError => ccc
+  rescue Redis::CannotConnectError
     logger.fatal "Can't connect to redis exiting now".red
     exit
   rescue Exception => e