sp-job 0.1.17 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dace51fb1d47771f51c07c8245a5190ef8a24ebb
-  data.tar.gz: 63e157d0906ae4678e0d0495864726d44f9b67ce
+  metadata.gz: 759aab586beb6fd8f184cd2649a53973fbc789f5
+  data.tar.gz: 1827e99f054c36f10130ab89958601ea59f19627
 SHA512:
-  metadata.gz: d8e740ec15d49f33aa7853fe45871c4bcae2b954c1a6655f7c781cb092f229dd77931f2cc7aa7bc34b76bc31317cc387d5a66d5458ae718869e17e150cbac250
-  data.tar.gz: a35ae11692e8d538533b7074dbb1521c03ee512e26e065dba80b2bba93c1c8af3c392e71140c4011a07ccb7d25ff966874f5e69bae69b9f004c3f83f034869a6
+  metadata.gz: 6311403a456ac54698ee3ecca61e7d9f192191b445bc338ce5e0496f6a7189ef8b7d11bbcf363c86d5f8f3f0bb3e4327663c17d0c6d879f1c96c08241b42a59f
+  data.tar.gz: 198cb7fd83ea854ed76c5b8195c0af534923d2ef46a4eedd275b7736f0b1740b041c3e5eda45b0be7df19dd7e2064a3ea3ba76e5d61ced7ee32991248710a0c6

data/Gemfile

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in sp-job.gemspec
 gemspec
 
-gem 'sp-duh', git: 'git@github.com:vpfpinho/sp-duh.git'
+#gem 'sp-duh', git: 'git@github.com:vpfpinho/sp-duh.git'

data/README.md

@@ -1,36 +1,59 @@
 # Sp::Job
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/sp/job`. To experiment with that code, run `bin/console` for an interactive prompt.
+# Execute the job
 
-TODO: Delete this and the text above, and describe your gem
+The job is executed method perform of the job class. If the tube is called 'park-fun' 
 
-## Installation
+```ruby
+  class ParkFun
 
-Add this line to your application's Gemfile:
+  def self.perform (job)
+
+  end
 
-```ruby
-gem 'sp-job'
 ```
 
-And then execute:
+## Return the job result
+
+use send_response(result: object)
+
+## Report a non-fatal error
+
+return report_error(message: 'i18n_message_key", args)
+
+## Report a fatal error
+
+Use raise_error(message:  'i18n_message_key", args)
+
+# Database access 
+
+Use the `db` object. 
+
+## db.exec
+
+The first argument is the query string followed by a variable number of arguments that are bound to the query.
+
+```ruby
+   db.exec('SELECT fun FROM public.park where id=$1', id_value)
+```
 
-    $ bundle
+Returns an xxx
 
-Or install it yourself as:
+## db.query
 
-    $ gem install sp-job
+# Redis
 
-## Usage
+Use the redis accessor to obtain a Redis object already connected 
 
-TODO: Write usage instructions here
+# Sending mails
 
-## Development
+Call send_mail
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+# Job configuration
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+Use config
 
-## Contributing
+# Logging 
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sp-job.
+Use logger
 

data/VERSION

@@ -1 +1 @@
-0.1.17
+0.2.2

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require "bundler/setup"
-require "sp/job"
+require "sp-job"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/bin/unique-file

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+#
+# Copyright (c) 2011-2018 Cloudware S.A. All rights reserved.
+#
+# This file is part of sp-job.
+#
+# sp-job is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# sp-job is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with sp-job.  If not, see <http://www.gnu.org/licenses/>.
+#
+# encoding: utf-8
+#
+
+
+require 'ap'
+require 'bundler/setup'
+require 'optparse'
+require 'sp/job/unique_file'
+require 'syslog/logger'
+
+begin
+
+  $args = {}
+  $config = {}
+
+  #
+  # Parse command line arguments
+  #
+  $option_parser = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$PROGRAM_NAME} ARGS"
+    opts.on('-p', '--path=PATH', 'Path to where file will be created') { |v| $args[:path] = v }
+    opts.on('-e', '--extension=EXTENSION', 'File extension')           { |v| $args[:ext]  = v } 
+  end
+  $option_parser.parse!
+
+  raise 'Must specify path' 	 if $args[:path].nil?
+  raise 'Must specify extension' if $args[:ext].nil?
+
+  raise 'Path not writable' unless File.writable?($args[:path])
+
+  file = SP::Job::Unique::File.create($args[:path], ".#{$args[:ext]}")
+
+  raise 'Could not create file'.red if file.nil?
+
+  puts file
+end

data/lib/sp-job.rb

@@ -24,22 +24,23 @@ require 'redis'
 require 'backburner'
 require 'json'
 require 'fileutils'
-require 'concurrent'
 require 'optparse'
 require 'os'
 require 'pg'
-require 'sp-duh'
-require 'oauth2'
-require 'oauth2-client'
-require 'curb'
-require 'rails'
+require 'oauth2'         unless RUBY_ENGINE == 'jruby'
+require 'oauth2-client'  unless RUBY_ENGINE == 'jruby'
+require 'curb'           unless RUBY_ENGINE == 'jruby'
 require 'erb'
 require 'ostruct'
 require 'json'
 require 'mail'
+require 'uri'
 
 require 'sp/job'
-require 'sp/job/engine'
 require 'sp/job/version'
 require 'sp/job/worker'
-require 'sp/job/broker_oauth2_client'
+require 'sp/job/worker_thread'
+require 'sp/job/common'
+require 'sp/job/unique_file'
+require 'sp/job/broker_http_client'      unless RUBY_ENGINE == 'jruby'
+require 'sp/job/broker_oauth2_client'    unless RUBY_ENGINE == 'jruby'

data/lib/sp/job/back_burner.rb

@@ -19,21 +19,75 @@
 # encoding: utf-8
 #
 require 'sp/job/pg_connection'
+require 'sp/job/job_db_adapter'
 require 'roadie'
+require 'thread'
+
+module SP
+  module Job
+
+    class JobCancelled < ::StandardError
+
+    end
+
+    class JobAborted < ::StandardError
+    end
+
+    class JobException < ::StandardError
+
+      attr_reader :job
+      attr_reader :args
+
+      def initialize (args:, job: nil)
+	      super(args[:message] || $current_job[:tube] || $args[:program_name])
+        @job     = job
+        @args    = args
+      end
+
+    end
+
+    class Logger < ::Logger
+
+      def task (sequence, text, success = true)
+        if success
+          info "[#{sequence}] #{text} \xE2\x9C\x94".green
+        else
+          info "[#{sequence}] #{text} \xE2\x9D\x8C".red
+        end
+      end
+    end
+
+    class ThreadData < Struct.new(:job_status, :report_time_stamp, :exception_reported, :job_id, :publish_key, :job_key, :current_job, :job_notification, :jsonapi)
+      def initialize
+        @job_status = {}
+        if $config[:options] && $config[:options][:jsonapi] == true
+          @jsonapi = SP::Duh::JSONAPI::Service.new($pg, nil, SP::Job::JobDbAdapter)
+        end
+      end
+    end
+
+    class FauxMutex
+      def synchronize (&block)
+        yield
+      end
+    end
+
+  end
+end
+
 
 #
 # Initialize global data needed for configuration
 #
-$prefix       = OS.mac? ? '/usr/local' : '/'
-$rollbar      = false
-$bury         = false
-$min_progress = 3 # TODO to config??
-$args         = {
-  stdout:       false,
-  log_level:    'info',
-  program_name: File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME)),
-  config_file:  File.join($prefix, 'etc', File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME)), 'conf.json'),
-  log_file:     File.join($prefix, 'var', 'log', 'jobs', "#{File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))}.log")
+$prefix           = OS.mac? ? '/usr/local' : ''
+$rollbar          = false
+$min_progress     = 3
+$args = {
+  stdout:           false,
+  log_level:        'info',
+  program_name:     File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME)),
+  config_file:      File.join($prefix, 'etc', File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME)), 'conf.json'),
+  default_log_file: File.join($prefix, 'var', 'log', 'jobs', "#{File.basename($PROGRAM_NAME, File.extname($PROGRAM_NAME))}.log")
 }
 
 #
@@ -41,17 +95,62 @@ $args         = {
 #
 $option_parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} ARGS"
-  opts.on('-c', '--config=CONFIG.JSON', "path to json configuration file (default: '#{$args[:config_file]}')") { |v| $args[:config_file] = File.expand_path(v) }
-  opts.on('-l', '--log=LOGFILE'       , "path to log file (default: '#{$args[:log_file]}')")                   { |v| $args[:log_file]    = File.expand_path(v) }
-  opts.on('-d', '--debug'             , "developer mode: log to stdout and print job")                         { $args[:debug]           = true                }
-  opts.on('-v', '--log_level=LEVEL'   , "Log level DEBUG, INFO, WARN, ERROR, FATAL")                           { |v| $args[:log_level]   = v                   }
+  opts.on('-c', '--config=CONFIG.JSON', "path to json configuration file (default: '#{$args[:default_log_file]}')") { |v| $args[:config_file] = File.expand_path(v) }
+  opts.on('-l', '--log=LOGFILE'       , "path to log file (default: '#{$args[:log_file]}')")                        { |v| $args[:log_file]    = File.expand_path(v) }
+  opts.on('-d', '--debug'             , "developer mode: log to stdout and print job")                              { $args[:debug]           = true                }
+  opts.on('-v', '--log_level=LEVEL'   , "Log level DEBUG, INFO, WARN, ERROR, FATAL")                                { |v| $args[:log_level]   = v                   }
+  opts.on('-i', '--index=IDX'         , "systemd instance index")                                                   { |v| $args[:index]       = v                   }
 end
 $option_parser.parse!
 
+if $args[:debug]
+  require 'ruby-debug' if RUBY_ENGINE == 'jruby'
+end
+
+# Adjust log file if need, user specified option always takes precedence
+#
+if $args[:log_file].nil?
+  if $args[:index].nil?
+    $args[:log_file] = $args[:default_log_file]
+  else
+    $args[:log_file] = File.join($prefix, 'var', 'log', 'jobs', "#{$args[:program_name]}.#{$args[:index]}.log")
+  end
+end
+
+#
+# Create PID file for this jobs instance
+#
+if  OS.mac?
+  Dir.mkdir("#{$prefix}/var/run/jobs") unless Dir.exist? "#{$prefix}/var/run/jobs"
+end
+File.write("#{$prefix}/var/run/jobs/#{$args[:program_name]}#{$args[:index].nil? ? '' : '.' + $args[:index]}.pid", Process.pid)
+
 #
 # Read configuration
 #
 $config = JSON.parse(File.read(File.expand_path($args[:config_file])), symbolize_names: true)
+$min_progress = $config[:options][:min_progress]
+
+#
+# Global data for mutex and sync
+#
+$threads = [ Thread.current ]
+$thread_data = {}
+$thread_data[Thread.current] = ::SP::Job::ThreadData.new
+
+#
+# Sanity check we only support multithreading on JRUBY
+#
+if $config[:options] && $config[:options][:threads].to_i > 1
+  raise 'Multithreading is not supported in MRI/CRuby' unless RUBY_ENGINE == 'jruby'
+  $redis_mutex = Mutex.new
+  $roolbar_mutex = Mutex.new
+  $multithreading = true
+else
+  $redis_mutex = nil
+  $roolbar_mutex = ::SP::Job::FauxMutex.new
+  $multithreading = false
+end
 
 #
 # Configure rollbar
@@ -69,25 +168,50 @@ end
 #
 Backburner.configure do |config|
 
-  config.beanstalk_url       = "beanstalk://#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
-  config.on_error            = lambda { |e|
-    if $exception_reported == false
-      $exception_reported == true
-      update_progress(status: 'error', message: e)
+  config.beanstalk_url = "beanstalk://#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
+  config.on_error      = lambda { |e|
+    td = thread_data
+    if td.exception_reported == false
+      td.exception_reported = true
+      if e.instance_of? Beaneater::DeadlineSoonError
+        logger.warn "got a deadline warning".red
+      else
+        begin
+          raise_error(message: e)
+        rescue => e
+          # Do not retrow!!!!
+        end
+      end
     end
-    if $rollbar
-      Rollbar.error(e)
+
+    # Report exception to rollbar
+    $roolbar_mutex.synchronize {
+      if $rollbar
+        if e.instance_of? ::SP::Job::JobException
+          e.job[:password] = '<redacted>'
+          Rollbar.error(e, e.message, { job: e.job, args: e.args})
+        else
+          Rollbar.error(e)
+        end
+      end
+    }
+
+    # Catch fatal exception that must be handled with a restarts (systemctl will restart us)
+    case e
+    when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
+      logger.fatal "Lost connection to database exiting now"
+      exit
+    when Redis::CannotConnectError
+      logger.fatal "Can't connect to redis exiting now"
+      exit
     end
-    catch_fatal_exceptions(e)
   }
-  #config.priority_labels     = { :custom => 50, :useless => 1000 }
-  #config.max_job_retries     = 0 # default 0 retries
-  #config.retry_delay         = 5 # default 5 seconds
-  #config.default_priority    = 65536
+  config.max_job_retries  = ($config[:options] && $config[:options][:max_job_retries]) ? $config[:options][:max_job_retries] : 0
+  config.retry_delay      = ($config[:options] && $config[:options][:retry_delay])     ? $config[:options][:retry_delay]     : 5
   config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
   config.respond_timeout  = 120
-  config.default_worker   = SP::Job::Worker
-  config.logger           = $args[:debug] ? Logger.new(STDOUT) : Logger.new($args[:log_file])
+  config.default_worker   = $config[:options] && $config[:options][:threads].to_i > 1 ? SP::Job::WorkerThread : SP::Job::Worker
+  config.logger           = $args[:debug] ? SP::Job::Logger.new(STDOUT) : SP::Job::Logger.new($args[:log_file])
   config.logger.formatter = proc do |severity, datetime, progname, msg|
     date_format = datetime.strftime("%Y-%m-%d %H:%M:%S")
     "[#{date_format}] #{severity}: #{msg}\n"
@@ -136,8 +260,42 @@ if $config[:mail]
 end
 
 #
-# Monkey patches to keep the tube name as plain vannila job name
+# Monkey patches the sad reality of ruby development
 #
+if RUBY_ENGINE == 'jruby'
+
+  class Logger
+    class LogDevice
+
+      def write(message)
+        begin
+          synchronize do
+            if @shift_age and @dev.respond_to?(:stat)
+              begin
+                check_shift_log
+              rescue
+                warn("log shifting failed. #{$!}")
+              end
+            end
+            begin
+              @dev.write(message)
+            rescue ::SP::Job::JobCancelled => jc
+              raise jc
+            rescue
+              warn("log writing failed. #{$!}")
+            end
+          end
+        rescue ::SP::Job::JobCancelled => jc
+          raise jc
+        rescue Exception => ignored
+          warn("log writing failed. #{ignored}")
+        end
+      end
+
+    end
+  end
+end
+
 module Backburner
   module Helpers
 
@@ -148,19 +306,49 @@ module Backburner
   end
 
   module Logger
-    def log_job_begin(name, args)
-      log_info "Work job #{name}"
-      @job_started_at = Time.now
+
+    if RUBY_ENGINE != 'jruby'
+
+      def log_job_begin(name, args)
+        log_info "Work job #{name}"
+        @job_started_at = Time.now
+      end
+
+    else
+
+      def log_job_begin(name, args)
+        log_info "Work job #{name}"
+        Thread.current[:job_started_at] = Time.now
+      end
+
+      # Print out when a job completed
+      # If message is nil, job is considered complete
+      def log_job_end(name, message = nil)
+        ellapsed = Time.now - Thread.current[:job_started_at]
+        ms = (ellapsed.to_f * 1000).to_i
+        action_word = message ? 'Finished' : 'Completed'
+        log_info("#{action_word} #{name} in #{ms}ms #{message}")
+      end
+
     end
   end
 
   class Job
+    extend SP::Job::Common # to bring in logger and report_error into this class
+
     # Processes a job and handles any failure, deleting the job once complete
     #
     # @example
     #   @task.process
     #
     def process
+      # Invoke the job setup function, bailout if the setup returns false
+      unless job_class.respond_to?(:prepare_job) && job_class.prepare_job(*args)
+        task.delete
+        logger.warn "Delete stale or preempted task".red
+        return false
+      end
+
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_class, :before_perform, *args)
       unless res
@@ -179,6 +367,41 @@ module Backburner
       task.delete
       # Invoke after perform hook
       @hooks.invoke_hook_events(job_class, :after_perform, *args)
+    rescue ::SP::Job::JobAborted => ja
+      #
+      # This exception:
+      #  1. is sent to the rollbar
+      #  2. does not bury the job, instead the job is deleted
+      #
+      logger.debug "Received job aborted exception #{Thread.current}".yellow
+      unless task.nil?
+        logger.debug 'Task deleted'.yellow
+        task.delete
+      end
+      # Invoke after perform hook
+      @hooks.invoke_hook_events(job_class, :after_perform, *args)
+      thread_data.job_id = nil
+    rescue ::SP::Job::JobCancelled => jc
+      #
+      # This exception:
+      #  1. is not sent to the rollbar
+      #  2. does not bury the job, instead the job is deleted
+      #
+      logger.debug "Received job cancellation exception #{Thread.current}".yellow
+      unless task.nil?
+        logger.debug 'Task deleted'.yellow
+        task.delete
+      end
+      @hooks.invoke_hook_events(job_class, :on_failure, jc, *args)
+      report_error(message: 'i18n_job_cancelled', status: 'cancelled')
+      if $redis_mutex.nil?
+        $redis.hset(thread_data.job_key, 'cancelled', true)
+      else
+        $redis_mutex.synchronize {
+          $redis.hset(thread_data.job_key, 'cancelled', true)
+        }
+      end
+      thread_data.job_id = nil
     rescue => e
       @hooks.invoke_hook_events(job_class, :on_failure, e, *args)
       raise e
@@ -186,20 +409,56 @@ module Backburner
   end
 end
 
-# Mix-in the mix-in in the script so that we can use the Common module functions
-require 'sp/job/common'
+# Mix-in the common mix-in to make code available for the lambdas used in this file
 extend SP::Job::Common
 
+logger.debug "Log file ... #{$args[:log_file]}"
+logger.debug "PID ........ #{Process.pid}"
+
 #
 # Now create the global data needed by the mix-in methods
 #
-$connected          = false
-$job_status         = {}
-$validity           = 2
-$redis              = Redis.new(:host => $config[:redis][:host], :port => $config[:redis][:port], :db => 0)
-$beaneater          = Beaneater.new "#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
-$check_db_life_span = false
-$status_dirty       = false
+$connected     = false
+$redis         = Redis.new(:host => $config[:redis][:host], :port => $config[:redis][:port], :db => 0)
+$transient_job = $config[:options] && $config[:options][:transient] == true
+$beaneater     = Beaneater.new "#{$config[:beanstalkd][:host]}:#{$config[:beanstalkd][:port]}"
 if $config[:postgres] && $config[:postgres][:conn_str]
-  $pg = ::SP::Job::PGConnection.new(owner: 'back_burner', config: $config[:postgres])
+  $pg = ::SP::Job::PGConnection.new(owner: $PROGRAM_NAME, config: $config[:postgres], multithreaded: $multithreading)
+  if $PROGRAM_NAME.split('/').last == 'saft-importer' || $PROGRAM_NAME.split('/').last == 'saft-destroyer'
+    $pg.exec("SET log_min_duration_statement TO 0;")
+  end
+  if $config[:options][:jsonapi] == true
+    $jsonapi = SP::Duh::JSONAPI::Service.new($pg, ($jsonapi.nil? ? nil : $jsonapi.url), SP::Job::JobDbAdapter)
+  end
 end
+
+#
+# Open a second thread that will listen to cancellation and other "signals"
+#
+$cancel_thread = Thread.new {
+  begin
+    $subscription_redis = Redis.new(:host => $config[:redis][:host], :port => $config[:redis][:port], :db => 0)
+    $subscription_redis.subscribe($config[:service_id] + ':job-signal') do |on|
+      on.message do |channel, msg|
+        begin
+          message = JSON.parse(msg, {symbolize_names: true})
+          $threads.each do |thread|
+            if $thread_data[thread].job_id != nil && message[:id].to_s == $thread_data[thread].job_id && message[:status] == 'cancelled'
+              logger.info "Received cancel signal for job #{$thread_data[thread].job_id}"
+              thread.raise(::SP::Job::JobCancelled.new)
+            end
+          end
+        rescue Exception => e
+          # ignore invalid payloads
+        end
+      end
+    end
+  rescue Redis::CannotConnectError => ccc
+    logger.fatal "Can't connect to redis exiting now".red
+    exit
+  rescue Exception => e
+    # Forward unexpected exceptions to the main thread for proper handling
+    logger.fatal e.to_s.red
+    Thread.main.raise(e)
+  end
+}