rocketjob 2.0.0.rc3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 272892209b4b987a90ec99fe808e7197588c1fea
-  data.tar.gz: 560ead0b08ab42a2fc6cda4de2f3ed107199fce7
+  metadata.gz: 3bf52fdcaa423858a798fdd0da28cf9df7448170
+  data.tar.gz: a8cd888f37215dbe633dad2f13941e629c6463dc
 SHA512:
-  metadata.gz: b7c162229dfaed223d3346f6a31613026fd8dff5dcbd590e11b9b4569e98b65ed9fa31850db395248d54be9005017cf7bb28c12517a559a38e84f44b90146656
-  data.tar.gz: 0614a2ccfdcece55f6480d880eef065e8034629695f7678e21bc6212231351c24ebf1cab06e53553f7102479554eca419eadd208a4716b7a83a53e25ab56bc46
+  metadata.gz: 99b6ff7af94d554ff2107deaaade0b639703cf29cd714fcdbe0763f1e4217ae8b22e87a790fd87101a849d1172684a670fb2c6b80c6ec5689ca3991795a0bb53
+  data.tar.gz: 1d477d2acc562df2a16ebccf500e71d77c570feb486c38a01f8bdcabacc99ebcdf56394b05049b893b5237754633920643c8e8387058666bc0f710984f9058f1

data/bin/rocketjob

@@ -8,7 +8,7 @@ begin
   RocketJob::CLI.new(ARGV).run
 rescue => exc
   # Failsafe logger that writes to STDERR
-  SemanticLogger.add_appender(STDERR, :error, &SemanticLogger::Appender::Base.colorized_formatter)
+  SemanticLogger.add_appender(io: STDERR, level: :error, formatter: :color)
   SemanticLogger['RocketJob'].error('Rocket Job shutting down due to exception', exc)
   SemanticLogger.flush
   exit 1

data/bin/rocketjob_perf

@@ -1,91 +1,10 @@
 #!/usr/bin/env ruby
-require 'csv'
-require 'yaml'
 require 'rocketjob'
 
 # Log to console
-SemanticLogger.add_appender(STDOUT, &SemanticLogger::Appender::Base.colorized_formatter)
+SemanticLogger.add_appender(io: STDOUT, formatter: :color)
 
-class RocketJobPerf
-  attr_accessor :count, :worker_processes, :worker_threads, :version, :ruby, :environment, :mongo_config
-
-  def initialize
-    @version          = RocketJob::VERSION
-    @ruby             = defined?(JRuby) ? "jruby_#{JRUBY_VERSION}" : "ruby_#{RUBY_VERSION}"
-    @count            = 100_000
-    @worker_processes = 0
-    @worker_threads   = 0
-    @environment      = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-    @mongo_config     = 'config/mongo.yml'
-  end
-
-  def run_test_case(count = self.count)
-    self.worker_processes = RocketJob::Worker.count
-    raise 'Please start workers before starting the performance test' if worker_processes == 0
-
-    self.worker_processes = 0
-    self.worker_threads   = 0
-    RocketJob::Worker.where(state: :running).each do |worker_process|
-      unless worker_process.zombie?
-        self.worker_processes += 1
-        self.worker_threads   += worker_process.heartbeat.current_threads
-      end
-    end
-    puts "Running: #{worker_threads} workers, distributed across #{worker_processes} processes"
-
-    puts 'Waiting for workers to pause'
-    RocketJob::Worker.pause_all
-    RocketJob::Jobs::SimpleJob.delete_all
-    sleep 15
-
-    puts 'Enqueuing jobs'
-    first = RocketJob::Jobs::SimpleJob.create(priority: 1, destroy_on_complete: false)
-    (count - 2).times { |i| RocketJob::Jobs::SimpleJob.create! }
-    last = RocketJob::Jobs::SimpleJob.create(priority: 100, destroy_on_complete: false)
-
-    puts 'Resuming workers'
-    RocketJob::Worker.resume_all
-
-    while (!last.reload.completed?)
-      sleep 3
-    end
-
-    duration = last.reload.completed_at - first.reload.started_at
-    {count: count, duration: duration, jobs_per_second: (count.to_f / duration).to_i}
-  end
-
-  # Export the Results hash to a CSV file
-  def export_results(results)
-    CSV.open("job_results_#{ruby}_#{worker_processes}p_#{threads}t_v#{version}.csv", 'wb') do |csv|
-      csv << results.first.keys
-      results.each { |result| csv << result.values }
-    end
-  end
-
-  # Parse command line options
-  def parse(argv)
-    parser        = OptionParser.new do |o|
-      o.on('-c', '--count COUNT', 'Count of jobs to enqueue') do |arg|
-        self.count = arg.to_i
-      end
-      o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Location of mongo.yml config file') do |arg|
-        self.mongo_config = arg
-      end
-      o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
-        self.environment = arg
-      end
-    end
-    parser.banner = 'rocketjob_perf <options>'
-    parser.on_tail '-h', '--help', 'Show help' do
-      puts parser
-      exit 1
-    end
-    parser.parse! argv
-  end
-
-end
-
-perf = RocketJobPerf.new
+perf = RocketJob::Performance.new
 perf.parse(ARGV)
 RocketJob::Config.load!(perf.environment, perf.mongo_config)
 results = perf.run_test_case

data/lib/rocket_job/cli.rb

@@ -25,6 +25,7 @@ module RocketJob
       setup_environment
       setup_logger
       rails? ? boot_rails : boot_standalone
+#      setup_metrics
       write_pidfile
 
       opts               = {}
@@ -33,6 +34,15 @@ module RocketJob
       Worker.run(opts)
     end
 
+    def setup_metrics
+      SemanticLogger.on_metric do |log|
+        if log.metric.start_with?('rocketjob/')
+          ap log
+          RocketJob::Stats::ShortTerm.increment_metric(log.time, log.name, log.duration, log.metric_amount)
+        end
+      end
+    end
+
     def rails?
       @rails ||= begin
         boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
@@ -58,7 +68,7 @@ module RocketJob
       SemanticLogger.default_level = log_level.to_sym if log_level
 
       if Rails.configuration.eager_load
-        RocketJob::Worker.logger.benchmark_info('Eager loaded Rails and all Engines') do
+        RocketJob::Worker.logger.measure_info('Eager loaded Rails and all Engines') do
           Rails.application.eager_load!
           Rails::Engine.subclasses.each(&:eager_load!)
         end
@@ -83,7 +93,7 @@ module RocketJob
       # Log to file except when booting rails, when it will add the log file path
       path = log_file ? Pathname.new(log_file) : Pathname.pwd.join("log/#{environment}.log")
       path.dirname.mkpath
-      SemanticLogger.add_appender(path.to_s, &SemanticLogger::Appender::Base.colorized_formatter)
+      SemanticLogger.add_appender(file_name: path.to_s, formatter: :color)
 
       logger.info "Rails not detected. Running standalone: #{environment}"
       RocketJob::Config.load!(environment)
@@ -112,7 +122,7 @@ module RocketJob
     end
 
     def setup_logger
-      SemanticLogger.add_appender(STDOUT, &SemanticLogger::Appender::Base.colorized_formatter) unless quiet
+      SemanticLogger.add_appender(io: STDOUT, formatter: :color) unless quiet
       SemanticLogger.default_level = log_level.to_sym if log_level
 
       # Enable SemanticLogger signal handling for this process

data/lib/rocket_job/dirmon_entry.rb

@@ -251,7 +251,7 @@ module RocketJob
 
     # Passes each filename [Pathname] found that matches the pattern into the supplied block
     def each(&block)
-      logger.fast_tag("DirmonEntry:#{id}") do
+      logger.fast_tag("dirmon_entry:#{id}") do
         # Case insensitive filename matching
         Pathname.glob(pattern, File::FNM_CASEFOLD).each do |pathname|
           next if pathname.directory?
@@ -305,7 +305,7 @@ module RocketJob
     # Queues the job for the supplied pathname
     def later(pathname)
       if klass = job_class
-        logger.benchmark_info "Enqueued: #{name}, Job class: #{job_class_name}" do
+        logger.measure_info "Enqueued: #{name}, Job class: #{job_class_name}" do
           job = klass.new(properties.merge(arguments: arguments))
           upload_file(job, pathname)
           job.save!

data/lib/rocket_job/extensions/mongo.rb

@@ -16,7 +16,7 @@ module Mongo
     private
 
     def log_operation(name, payload, duration)
-      MongoClient.logger.benchmark_trace(name, duration: (duration * 1000), payload: payload)
+      MongoClient.logger.measure_trace(name, duration: (duration * 1000), payload: payload)
     end
 
   end

data/lib/rocket_job/performance.rb

@@ -0,0 +1,82 @@
+require 'csv'
+require 'yaml'
+module RocketJob
+  class Performance
+    attr_accessor :count, :worker_processes, :worker_threads, :version, :ruby, :environment, :mongo_config
+
+    def initialize
+      @version          = RocketJob::VERSION
+      @ruby             = defined?(JRuby) ? "jruby_#{JRUBY_VERSION}" : "ruby_#{RUBY_VERSION}"
+      @count            = 100_000
+      @worker_processes = 0
+      @worker_threads   = 0
+      @environment      = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      @mongo_config     = 'config/mongo.yml'
+    end
+
+    def run_test_case(count = self.count)
+      self.worker_processes = RocketJob::Worker.count
+      raise 'Please start workers before starting the performance test' if worker_processes == 0
+
+      self.worker_processes = 0
+      self.worker_threads   = 0
+      RocketJob::Worker.where(state: :running).each do |worker_process|
+        unless worker_process.zombie?
+          self.worker_processes += 1
+          self.worker_threads   += worker_process.heartbeat.current_threads
+        end
+      end
+      puts "Running: #{worker_threads} workers, distributed across #{worker_processes} processes"
+
+      puts 'Waiting for workers to pause'
+      RocketJob::Worker.pause_all
+      RocketJob::Jobs::SimpleJob.delete_all
+      sleep 15
+
+      puts 'Enqueuing jobs'
+      first = RocketJob::Jobs::SimpleJob.create(priority: 1, destroy_on_complete: false)
+      (count - 2).times { |i| RocketJob::Jobs::SimpleJob.create! }
+      last = RocketJob::Jobs::SimpleJob.create(priority: 100, destroy_on_complete: false)
+
+      puts 'Resuming workers'
+      RocketJob::Worker.resume_all
+
+      while (!last.reload.completed?)
+        sleep 3
+      end
+
+      duration = last.reload.completed_at - first.reload.started_at
+      {count: count, duration: duration, jobs_per_second: (count.to_f / duration).to_i}
+    end
+
+    # Export the Results hash to a CSV file
+    def export_results(results)
+      CSV.open("job_results_#{ruby}_#{worker_processes}p_#{threads}t_v#{version}.csv", 'wb') do |csv|
+        csv << results.first.keys
+        results.each { |result| csv << result.values }
+      end
+    end
+
+    # Parse command line options
+    def parse(argv)
+      parser        = OptionParser.new do |o|
+        o.on('-c', '--count COUNT', 'Count of jobs to enqueue') do |arg|
+          self.count = arg.to_i
+        end
+        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Location of mongo.yml config file') do |arg|
+          self.mongo_config = arg
+        end
+        o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
+          self.environment = arg
+        end
+      end
+      parser.banner = 'rocketjob_perf <options>'
+      parser.on_tail '-h', '--help', 'Show help' do
+        puts parser
+        exit 1
+      end
+      parser.parse! argv
+    end
+
+  end
+end

data/lib/rocket_job/plugins/cron.rb

@@ -0,0 +1,130 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    # Automatically schedules the job to start based on the supplied `cron_schedule`.
+    # Once started the job will automatically restart on completion and will only run again
+    # according to the `cron_schedule`.
+    # Failed jobs are aborted so that they cannot be restarted since a new instance has already
+    # been enqueued.
+    #
+    # Include RocketJob::Plugins::Singleton to prevent multiple copies of the job from running at
+    # the same time.
+    #
+    # Unlike cron, if a job is already running, another one is not queued when the cron
+    # schedule needs another started, but rather on completion of the current job. This prevents
+    # multiple instances of the same job from running at the same time. The next instance of the
+    # job is only scheduled on completion of the current job instance.
+    #
+    # For example if the job takes 10 minutes to complete, and is scheduled to run every 5 minutes,
+    # it will only be run every 10 minutes.
+    #
+    # Their is no centralized scheduler or need to start schedulers anywhere, since the jobs
+    # can be picked up by any Rocket Job worker. Once processing is complete a new instance is then
+    # automatically scheduled based on the `cron_schedule`.
+    #
+    # A job is only queued to run at the specified `cron_schedule`, it will only run if there are workers
+    # available to run. For example if workers are busy working on higher priority jobs, then the job
+    # will only run once those jobs have completed, or their priority lowered. Additionally, while the
+    # job is queued no additional instances will be enqueued, even if the next cron interval has been reached.
+    #
+    # Note:
+    # - The job will not be restarted if:
+    #   - A validation fails after cloning this job.
+    #   - The job has expired.
+    #
+    # Example:
+    #
+    # class MyCronJob < RocketJob::Job
+    #   include RocketJob::Plugins::Cron
+    #
+    #   # Set the default cron_schedule
+    #   rocket_job do |job|
+    #     job.cron_schedule = "* 1 * * * UTC"
+    #   end
+    #
+    #   def perform
+    #     puts "DONE"
+    #   end
+    # end
+    #
+    # # Queue the job for processing using the default cron_schedule specified above
+    # MyCronJob.create!
+    #
+    # # Set the cron schedule:
+    # MyCronJob.create!(cron_schedule: '* 1 * * * America/New_York')
+    #
+    #
+    # Note:
+    #
+    # To prevent multiple instances of the job from running at the same time,
+    # add: "include RocketJob::Plugins::Singleton"
+    #
+    # Example: Only allow one instance of the cron job to run at a time:
+    #
+    # class MyCronJob < RocketJob::Job
+    #   # Add `cron_schedule`
+    #   include RocketJob::Plugins::Cron
+    #   # Prevents mutiple instances from being queued or run at the same time
+    #   include RocketJob::Plugins::Singleton
+    #
+    #   # Set the default cron_schedule
+    #   rocket_job do |job|
+    #     job.cron_schedule = "* 1 * * * UTC"
+    #   end
+    #
+    #   def perform
+    #     puts "DONE"
+    #   end
+    # end
+    #
+    # Note: The cron_schedule field is formatted as follows:
+    #
+    #     *    *    *    *    *    *
+    #     ┬    ┬    ┬    ┬    ┬    ┬
+    #     │    │    │    │    │    │
+    #     │    │    │    │    │    └ Optional: Timezone, for example: 'America/New_York', 'UTC'
+    #     │    │    │    │    └───── day_of_week (0-7) (0 or 7 is Sun, or use 3-letter names)
+    #     │    │    │    └────────── month (1-12, or use 3-letter names)
+    #     │    │    └─────────────── day_of_month (1-31)
+    #     │    └──────────────────── hour (0-23)
+    #     └───────────────────────── minute (0-59)
+    #
+    # * When specifying day of week, both day 0 and day 7 is Sunday.
+    # * Ranges & Lists of numbers are allowed.
+    # * Ranges or lists of names are not allowed.
+    # * Ranges can include 'steps', so `1-9/2` is the same as `1,3,5,7,9`.
+    # * Months or days of the week can be specified by name.
+    # * Use the first three letters of the particular day or month (case doesn't matter).
+    # * The timezone is recommended to prevent any issues with possible default timezone
+    #   differences across servers, or environments.
+    module Cron
+      extend ActiveSupport::Concern
+
+      included do
+        include Restart
+
+        key :cron_schedule, String
+
+        before_create :rocket_job_set_run_at
+
+        validates_presence_of :cron_schedule
+        validates_each :cron_schedule do |record, attr, value|
+          begin
+            RocketJob::Plugins::Rufus::CronLine.new(value)
+          rescue ArgumentError => exc
+            record.errors.add(attr, exc.message)
+          end
+        end
+      end
+
+      private
+
+      def rocket_job_set_run_at
+        self.run_at = RocketJob::Plugins::Rufus::CronLine.new(cron_schedule).next_time
+      end
+
+    end
+  end
+end

data/lib/rocket_job/plugins/job/logger.rb

@@ -20,7 +20,7 @@ module RocketJob
         #   - silence noisy jobs by raising log level
         def rocket_job_around_logger(&block)
           logger.info('Start #perform')
-          logger.benchmark_info(
+          logger.measure_info(
             'Completed #perform',
             metric:             "rocketjob/#{self.class.name.underscore}/perform",
             log_exception:      :full,

data/lib/rocket_job/plugins/processing_window.rb

@@ -0,0 +1,88 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    # Ensure that a job will only run between certain hours of the day, regardless of when it was
+    # created/enqueued. Useful for creating a job now that should only be processed later during a
+    # specific time window. If the time window is already active the job is able to be processed
+    # immediately.
+    #
+    # Example: Process this job on Monday’s between 8am and 10am.
+    #
+    # Example: Run this job on the 1st of every month from midnight for the entire day.
+    #
+    # Since the cron schedule supports time zones it is easy to setup jobs to run at UTC or any other time zone.
+    #
+    # Example:
+    # # Only run the job between the hours of 8:30am and 8:30pm. If it is after 8:30pm schedule
+    # # it to run at 8:30am the next day.
+    # class BusinessHoursJob < RocketJob::Job
+    #   include RocketJob::Plugins::ProcessingWindow
+    #
+    #   # Set the default processing_window
+    #   rocket_job do |job|
+    #     # The start of the processing window
+    #     job.processing_schedule = "30 8 * * * America/New_York"
+    #     # How long the processing window is:
+    #     job.processing_duration = 12.hours
+    #   end
+    #
+    #   def perform
+    #     # Job will only run between 8:30am and 8:30pm Eastern
+    #   end
+    # end
+    #
+    # Note:
+    #   If a job is created/enqueued during the processing window, but due to busy/unavailable workers
+    #   is not processed during the window, the current job will be re-queued for the beginning
+    #   of the next processing window.
+    module ProcessingWindow
+      extend ActiveSupport::Concern
+
+      included do
+        key :processing_schedule, String
+        key :processing_duration, Integer
+
+        before_create :rocket_job_processing_window_set_run_at
+        before_retry :rocket_job_processing_window_set_run_at
+        after_start :rocket_job_processing_window_check
+
+        validates_presence_of :processing_schedule, :processing_duration
+        validates_each :processing_schedule do |record, attr, value|
+          begin
+            RocketJob::Plugins::Rufus::CronLine.new(value)
+          rescue ArgumentError => exc
+            record.errors.add(attr, exc.message)
+          end
+        end
+      end
+
+      # Returns [true|false] whether this job is currently inside its processing window
+      def rocket_job_processing_window_active?
+        time          = Time.now
+        previous_time = rocket_job_processing_schedule.previous_time(time)
+        # Inside previous processing window?
+        previous_time + processing_duration > time
+      end
+
+      private
+
+      # Only process this job if it is still in its processing window
+      def rocket_job_processing_window_check
+        return if rocket_job_processing_window_active?
+        logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{rocket_job_processing_schedule.next_time}")
+        self.worker_name ||= 'inline'
+        self.requeue!(worker_name)
+      end
+
+      def rocket_job_processing_window_set_run_at
+        self.run_at = rocket_job_processing_schedule.next_time unless rocket_job_processing_window_active?
+      end
+
+      def rocket_job_processing_schedule
+        RocketJob::Plugins::Rufus::CronLine.new(processing_schedule)
+      end
+    end
+  end
+end