rocketjob 5.2.0.beta2 → 5.3.2

data/lib/rocket_job/sliced/slice.rb

@@ -25,8 +25,8 @@ module RocketJob
 
       # The record number of the first record in this slice.
       # Useful in knowing the line number of each record in this slice
-      # relative to the original file that was uploaded.
-      field :first_record_number, type: Integer
+      # relative to the original file that was uploaded. (1-based index)
+      field :first_record_number, type: Integer, default: 1
 
       #
       # Read-only attributes
@@ -41,7 +41,7 @@ module RocketJob
       # Number of times that this job has failed to process
       field :failure_count, type: Integer
 
-      # Number of the record within this slice (not the entire file/job) currently being processed. (One based index)
+      # Number of the record within this slice (not the entire file/job) currently being processed. (1-based index)
       field :processing_record_number, type: Integer
 
       # This name of the worker that this job is being processed by, or was processed by
@@ -112,7 +112,7 @@ module RocketJob
 
       # Returns [Integer] the record number of the record currently being processed relative to the entire file.
       def current_record_number
-        first_record_number.to_i + processing_record_number.to_i
+        first_record_number + (processing_record_number || 1) - 1
       end
 
       # Before Fail save the exception to this slice.

data/lib/rocket_job/subscribers/server.rb

@@ -9,16 +9,22 @@ module RocketJob
         @supervisor = supervisor
       end
 
-      def kill(server_id: nil, name: nil, wait_timeout: 3)
+      def kill(server_id: nil, name: nil, wait_timeout: 5)
         return unless my_server?(server_id, name)
 
         supervisor.synchronize do
+          Supervisor.shutdown!
+
+          supervisor.logger.info("Stopping Pool")
           supervisor.worker_pool.stop
-          supervisor.worker_pool.join(wait_timeout)
+          unless supervisor.worker_pool.living_count == 0
+            supervisor.logger.info("Giving pool #{wait_timeout} seconds to terminate")
+            sleep(wait_timeout)
+          end
+          supervisor.logger.info("Kill Pool")
           supervisor.worker_pool.kill
         end
 
-        Supervisor.shutdown!
         logger.info "Killed"
       end
 

data/lib/rocket_job/supervisor.rb

@@ -55,7 +55,9 @@ module RocketJob
 
     def stop!
       server.stop! if server.may_stop?
-      worker_pool.stop
+      synchronize do
+        worker_pool.stop
+      end
       until worker_pool.join
         logger.info "Waiting for workers to finish processing ..."
         # One or more workers still running so update heartbeat so that server reports "alive".

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "5.2.0.beta2".freeze
+  VERSION = "5.3.2".freeze
 end

data/lib/rocket_job/worker_pool.rb

@@ -61,6 +61,7 @@ module RocketJob
     # Kill Worker threads
     def kill
       workers.each(&:kill)
+      workers.clear
     end
 
     # Wait for all workers to stop.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.2.0.beta2
+  version: 5.3.2
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-06 00:00:00.000000000 Z
+date: 2020-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: fugit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 description: 
 email: 
 executables:
@@ -124,6 +138,7 @@ files:
 - lib/rocket_job/batch/tabular/output.rb
 - lib/rocket_job/batch/throttle.rb
 - lib/rocket_job/batch/throttle_running_workers.rb
+- lib/rocket_job/batch/throttle_windows.rb
 - lib/rocket_job/batch/worker.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
@@ -162,8 +177,6 @@ files:
 - lib/rocket_job/plugins/processing_window.rb
 - lib/rocket_job/plugins/restart.rb
 - lib/rocket_job/plugins/retry.rb
-- lib/rocket_job/plugins/rufus/cron_line.rb
-- lib/rocket_job/plugins/rufus/zo_time.rb
 - lib/rocket_job/plugins/singleton.rb
 - lib/rocket_job/plugins/state_machine.rb
 - lib/rocket_job/plugins/transaction.rb
@@ -207,11 +220,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Ruby's missing batch processing system.

data/lib/rocket_job/plugins/rufus/cron_line.rb

@@ -1,520 +0,0 @@
-#--
-# Copyright (c) 2006-2017, John Mettraux, jmettraux@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# Made in Japan.
-#++
-#@formatter:off
-
-require 'set'
-
-module RocketJob::Plugins::Rufus
-
-  #
-  # A 'cron line' is a line in the sense of a crontab
-  # (man 5 crontab) file line.
-  #
-  class CronLine
-
-    # The max number of years in the future or the past before giving up
-    # searching for #next_time or #previous_time respectively
-    #
-    NEXT_TIME_MAX_YEARS = 14
-
-    # The string used for creating this cronline instance.
-    #
-    attr_reader :original
-    attr_reader :original_timezone
-
-    attr_reader :seconds
-    attr_reader :minutes
-    attr_reader :hours
-    attr_reader :days
-    attr_reader :months
-    #attr_reader :monthdays # reader defined below
-    attr_reader :weekdays
-    attr_reader :timezone
-
-    def initialize(line)
-
-      fail ArgumentError.new(
-        "not a string: #{line.inspect}"
-      ) unless line.is_a?(String)
-
-      @original = line
-      @original_timezone = nil
-
-      items = line.split
-
-      if @timezone = RocketJob::Plugins::Rufus::ZoTime.get_tzone(items.last)
-        @original_timezone = items.pop
-      else
-        @timezone = RocketJob::Plugins::Rufus::ZoTime.get_tzone(:current)
-      end
-
-      fail ArgumentError.new(
-        "not a valid cronline : '#{line}'"
-      ) unless items.length == 5 or items.length == 6
-
-      offset = items.length - 5
-
-      @seconds = offset == 1 ? parse_item(items[0], 0, 59) : [ 0 ]
-      @minutes = parse_item(items[0 + offset], 0, 59)
-      @hours = parse_item(items[1 + offset], 0, 24)
-      @days = parse_item(items[2 + offset], -30, 31)
-      @months = parse_item(items[3 + offset], 1, 12)
-      @weekdays, @monthdays = parse_weekdays(items[4 + offset])
-
-      [ @seconds, @minutes, @hours, @months ].each do |es|
-
-        fail ArgumentError.new(
-          "invalid cronline: '#{line}'"
-        ) if es && es.find { |e| ! e.is_a?(Integer) }
-      end
-
-      if @days && @days.include?(0) # gh-221
-
-        fail ArgumentError.new('invalid day 0 in cronline')
-      end
-    end
-
-    # Returns true if the given time matches this cron line.
-    #
-    def matches?(time)
-
-      # FIXME Don't create a new ZoTime if time is already a ZoTime in same
-      #       zone ...
-      #       Wait, this seems only used in specs...
-      t = ZoTime.new(time.to_f, @timezone)
-
-      return false unless sub_match?(t, :sec, @seconds)
-      return false unless sub_match?(t, :min, @minutes)
-      return false unless sub_match?(t, :hour, @hours)
-      return false unless date_match?(t)
-      true
-    end
-
-    # Returns the next time that this cron line is supposed to 'fire'
-    #
-    # This is raw, 3 secs to iterate over 1 year on my macbook :( brutal.
-    # (Well, I was wrong, takes 0.001 sec on 1.8.7 and 1.9.1)
-    #
-    # This method accepts an optional Time parameter. It's the starting point
-    # for the 'search'. By default, it's Time.now
-    #
-    # Note that the time instance returned will be in the same time zone that
-    # the given start point Time (thus a result in the local time zone will
-    # be passed if no start time is specified (search start time set to
-    # Time.now))
-    #
-    #   Rufus::Scheduler::CronLine.new('30 7 * * *').next_time(
-    #     Time.mktime(2008, 10, 24, 7, 29))
-    #   #=> Fri Oct 24 07:30:00 -0500 2008
-    #
-    #   Rufus::Scheduler::CronLine.new('30 7 * * *').next_time(
-    #     Time.utc(2008, 10, 24, 7, 29))
-    #   #=> Fri Oct 24 07:30:00 UTC 2008
-    #
-    #   Rufus::Scheduler::CronLine.new('30 7 * * *').next_time(
-    #     Time.utc(2008, 10, 24, 7, 29)).localtime
-    #   #=> Fri Oct 24 02:30:00 -0500 2008
-    #
-    # (Thanks to K Liu for the note and the examples)
-    #
-    def next_time(from=ZoTime.now)
-
-      nt = nil
-      zt = ZoTime.new(from.to_i + 1, @timezone)
-      maxy = from.year + NEXT_TIME_MAX_YEARS
-
-      loop do
-
-        nt = zt.dup
-
-        fail RangeError.new(
-          "failed to reach occurrence within " +
-            "#{NEXT_TIME_MAX_YEARS} years for '#{original}'"
-        ) if nt.year > maxy
-
-        unless date_match?(nt)
-          zt.add((24 - nt.hour) * 3600 - nt.min * 60 - nt.sec)
-          next
-        end
-        unless sub_match?(nt, :hour, @hours)
-          zt.add((60 - nt.min) * 60 - nt.sec)
-          next
-        end
-        unless sub_match?(nt, :min, @minutes)
-          zt.add(60 - nt.sec)
-          next
-        end
-        unless sub_match?(nt, :sec, @seconds)
-          zt.add(next_second(nt))
-          next
-        end
-
-        break
-      end
-
-      nt
-    end
-
-    # Returns the previous time the cronline matched. It's like next_time, but
-    # for the past.
-    #
-    def previous_time(from=ZoTime.now)
-
-      pt = nil
-      zt = ZoTime.new(from.to_i - 1, @timezone)
-      miny = from.year - NEXT_TIME_MAX_YEARS
-
-      loop do
-
-        pt = zt.dup
-
-        fail RangeError.new(
-          "failed to reach occurrence within " +
-            "#{NEXT_TIME_MAX_YEARS} years for '#{original}'"
-        ) if pt.year < miny
-
-        unless date_match?(pt)
-          zt.substract(pt.hour * 3600 + pt.min * 60 + pt.sec + 1)
-          next
-        end
-        unless sub_match?(pt, :hour, @hours)
-          zt.substract(pt.min * 60 + pt.sec + 1)
-          next
-        end
-        unless sub_match?(pt, :min, @minutes)
-          zt.substract(pt.sec + 1)
-          next
-        end
-        unless sub_match?(pt, :sec, @seconds)
-          zt.substract(prev_second(pt))
-          next
-        end
-
-        break
-      end
-
-      pt
-    end
-
-    # Returns an array of 6 arrays (seconds, minutes, hours, days,
-    # months, weekdays).
-    # This method is mostly used by the cronline specs.
-    #
-    def to_a
-
-      [
-        toa(@seconds),
-        toa(@minutes),
-        toa(@hours),
-        toa(@days),
-        toa(@months),
-        toa(@weekdays),
-        toa(@monthdays),
-        @timezone.name
-      ]
-    end
-    alias to_array to_a
-
-    # Returns a quickly computed approximation of the frequency for this
-    # cron line.
-    #
-    # #brute_frequency, on the other hand, will compute the frequency by
-    # examining a whole year, that can take more than seconds for a seconds
-    # level cron...
-    #
-    def frequency
-
-      return brute_frequency unless @seconds && @seconds.length > 1
-
-      secs = toa(@seconds)
-
-      secs[1..-1].inject([ secs[0], 60 ]) { |(prev, delta), sec|
-        d = sec - prev
-        [ sec, d < delta ? d : delta ]
-      }[1]
-    end
-
-    # Caching facility. Currently only used for brute frequencies.
-    #
-    @cache = {}; class << self; attr_reader :cache; end
-
-    # Returns the shortest delta between two potential occurences of the
-    # schedule described by this cronline.
-    #
-    # .
-    #
-    # For a simple cronline like "*/5 * * * *", obviously the frequency is
-    # five minutes. Why does this method look at a whole year of #next_time ?
-    #
-    # Consider "* * * * sun#2,sun#3", the computed frequency is 1 week
-    # (the shortest delta is the one between the second sunday and the third
-    # sunday). This method takes no chance and runs next_time for the span
-    # of a whole year and keeps the shortest.
-    #
-    # Of course, this method can get VERY slow if you call on it a second-
-    # based cronline...
-    #
-    def brute_frequency
-
-      key = "brute_frequency:#{@original}"
-
-      delta = self.class.cache[key]
-      return delta if delta
-
-      delta = 366 * DAY_S
-
-      t0 = previous_time(Time.local(2000, 1, 1))
-
-      loop do
-
-        break if delta <= 1
-        break if delta <= 60 && @seconds && @seconds.size == 1
-
-        t1 = next_time(t0)
-        d = t1 - t0
-        delta = d if d < delta
-        break if @months.nil? && t1.month == 2
-        break if @months.nil? && @days.nil? && t1.day == 2
-        break if @months.nil? && @days.nil? && @hours.nil? && t1.hour == 1
-        break if @months.nil? && @days.nil? && @hours.nil? && @minutes.nil? && t1.min == 1
-        break if t1.year >= 2001
-
-        t0 = t1
-      end
-
-      self.class.cache[key] = delta
-    end
-
-    def next_second(time)
-
-      secs = toa(@seconds)
-
-      return secs.first + 60 - time.sec if time.sec > secs.last
-
-      secs.shift while secs.first < time.sec
-
-      secs.first - time.sec
-    end
-
-    def prev_second(time)
-
-      secs = toa(@seconds)
-
-      return time.sec + 60 - secs.last if time.sec < secs.first
-
-      secs.pop while time.sec < secs.last
-
-      time.sec - secs.last
-    end
-
-    protected
-
-    def sc_sort(a)
-
-      a.sort_by { |e| e.is_a?(String) ? 61 : e.to_i }
-    end
-
-    if RUBY_VERSION >= '1.9'
-      def toa(item)
-        item == nil ? nil : item.to_a
-      end
-    else
-      def toa(item)
-        item.is_a?(Set) ? sc_sort(item.to_a) : item
-      end
-    end
-
-    WEEKDAYS = %w[ sun mon tue wed thu fri sat ]
-    DAY_S = 24 * 3600
-
-    def parse_weekdays(item)
-
-      return nil if item == '*'
-
-      weekdays = nil
-      monthdays = nil
-
-      item.downcase.split(',').each do |it|
-
-        WEEKDAYS.each_with_index { |a, i| it.gsub!(/#{a}/, i.to_s) }
-
-        it = it.gsub(/([^#])l/, '\1#-1')
-        # "5L" == "5#-1" == the last Friday
-
-        if m = it.match(/\A(.+)#(l|-?[12345])\z/)
-
-          fail ArgumentError.new(
-            "ranges are not supported for monthdays (#{it})"
-          ) if m[1].index('-')
-
-          it = it.gsub(/#l/, '#-1')
-
-          (monthdays ||= []) << it
-
-        else
-
-          fail ArgumentError.new(
-            "invalid weekday expression (#{item})"
-          ) if it !~ /\A0*[0-7](-0*[0-7])?\z/
-
-          its = it.index('-') ? parse_range(it, 0, 7) : [ Integer(it) ]
-          its = its.collect { |i| i == 7 ? 0 : i }
-
-          (weekdays ||= []).concat(its)
-        end
-      end
-
-      weekdays = weekdays.uniq.sort if weekdays
-
-      [ weekdays, monthdays ]
-    end
-
-    def parse_item(item, min, max)
-
-      return nil if item == '*'
-
-      r = item.split(',').map { |i| parse_range(i.strip, min, max) }.flatten
-
-      fail ArgumentError.new(
-        "found duplicates in #{item.inspect}"
-      ) if r.uniq.size < r.size
-
-      r = sc_sort(r)
-
-      Set.new(r)
-    end
-
-    RANGE_REGEX = /\A(\*|-?\d{1,2})(?:-(-?\d{1,2}))?(?:\/(\d{1,2}))?\z/
-
-    def parse_range(item, min, max)
-
-      return %w[ L ] if item == 'L'
-
-      item = '*' + item if item[0, 1] == '/'
-
-      m = item.match(RANGE_REGEX)
-
-      fail ArgumentError.new(
-        "cannot parse #{item.inspect}"
-      ) unless m
-
-      mmin = min == -30 ? 1 : min # days
-
-      sta = m[1]
-      sta = sta == '*' ? mmin : sta.to_i
-
-      edn = m[2]
-      edn = edn ? edn.to_i : sta
-      edn = max if m[1] == '*'
-
-      inc = m[3]
-      inc = inc ? inc.to_i : 1
-
-      fail ArgumentError.new(
-        "#{item.inspect} positive/negative ranges not allowed"
-      ) if (sta < 0 && edn > 0) || (sta > 0 && edn < 0)
-
-      fail ArgumentError.new(
-        "#{item.inspect} descending day ranges not allowed"
-      ) if min == -30 && sta > edn
-
-      fail ArgumentError.new(
-        "#{item.inspect} is not in range #{min}..#{max}"
-      ) if sta < min || edn > max
-
-      fail ArgumentError.new(
-        "#{item.inspect} increment must be greater than zero"
-      ) if inc == 0
-
-      r = []
-      val = sta
-
-      loop do
-        v = val
-        v = 0 if max == 24 && v == 24 # hours
-        r << v
-        break if inc == 1 && val == edn
-        val += inc
-        break if inc > 1 && val > edn
-        val = min if val > max
-      end
-
-      r.uniq
-    end
-
-    # FIXME: Eventually split into day_match?, hour_match? and monthdays_match?o
-    #
-    def sub_match?(time, accessor, values)
-
-      return true if values.nil?
-
-      value = time.send(accessor)
-
-      if accessor == :day
-
-        values.each do |v|
-          return true if v == 'L' && (time + DAY_S).day == 1
-          return true if v.to_i < 0 && (time + (1 - v) * DAY_S).day == 1
-        end
-      end
-
-      if accessor == :hour
-
-        return true if value == 0 && values.include?(24)
-      end
-
-      if accessor == :monthdays
-
-        return true if (values & value).any?
-      end
-
-      values.include?(value)
-    end
-
-    #    def monthday_match?(zt, values)
-    #
-    #      return true if values.nil?
-    #
-    #      today_values = monthdays(zt)
-    #
-    #      (today_values & values).any?
-    #    end
-
-    def date_match?(zt)
-
-      return false unless sub_match?(zt, :day, @days)
-      return false unless sub_match?(zt, :month, @months)
-
-      return true if (
-      (@weekdays && @monthdays) &&
-        (sub_match?(zt, :wday, @weekdays) ||
-          sub_match?(zt, :monthdays, @monthdays)))
-
-      return false unless sub_match?(zt, :wday, @weekdays)
-      return false unless sub_match?(zt, :monthdays, @monthdays)
-
-      true
-    end
-  end
-end