queue-processor 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 88481dcefd871ebdea1fe2286bffe0aa125a379dd219753f28e21959b792e59b
+  data.tar.gz: 86d6a3baa440f8d217f2333fde259e09ba5e7c92bb67f2eb873f73ab438e2ca9
+SHA512:
+  metadata.gz: bcb8b7cb69e2e0b593287da839c12c000ac8479644a85af9af2b9e1d47f4e2b036fbd0ac3b284ee21defd255e56c540e81ada695593024d3cc15e85f2419120f
+  data.tar.gz: bf17501f746ca8d457ee7f965a4e468653b8419013cb12c3de31787692f9ce2de0368998cef0d182247fef43a2fcf0f2a5291b05275bf3b0929af6b1cd0c36fe

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,103 @@
+queue-processor
+===============
+Queue Processor is a calculation model built on top of Delayed Job designed to process a single root calculation followed by a number of dependent calculations.
+
+Installation
+============
+
+Add queue-processor and state_machine to your Gemfile.  Note that a specific version of state_machine is required until https://github.com/pluginaweek/state_machine/pull/255 is resolved.
+
+```ruby
+gem 'queue-processor', :git => 'https://github.com/GoodMeasuresLLC/queue-processor.git'
+gem 'state_machine', :git => 'https://github.com/GoodMeasuresLLC/state_machine.git'
+```
+
+Run `bundle install`
+
+Requirements
+============
+You must have 3 models: 
+
+1. The RootCalculation: This is the calculation that everything depends on.  Performing a new RootCalculation will cause everything in progress for the old calculation to be aborted.  For example, a Weekly Calculation that determine's the user's score for the week against their goals.
+2. A DependentCalculationGroup: These dependent calculations are queued and processed after the RootCalculation is performed.  DependentCalculationGroups simply queue additional work, partitioned and prioritized.  For example, a DailyCalculation could queue HourlyCalculations prioritized such that tomorrow's calculations are prioritized first.  When all the HourlyCalculations complete, they are made available as a group though the DailyCalculation.  When the last DailyCalculation completes, the RootCalculation is marked as finished.
+3. A DependentCalculation: A calculation that depends on the Root Calculation being performed.  DependentCalculations are made available in groups and when all are finished, the RootCalculation is marked as complete.  For example, an HourlyCalculation could be performed for a particular hour and day of the week.  The would be grouped by day of the week and when all the hourly calculations for a day are complete, that week's dependent calculation group would be marked as complete.  And when all the hours for the whole week are done, the Root Calculation would be marked as complete.
+
+Your models implement methods for performing the calculations and queuing dependent work
+
+Examples
+========
+
+```ruby
+class WeeklyCalculation < ActiveRecord::Base
+  include QueueProcessor::RootCalculation
+  root_calculation_config do
+    self.dependent_calculation_group_association = :daily_calculations
+    self.run_at = lambda {|run| Time.now + 0.005.seconds}
+    self.priority = lambda {|run| 0}
+  end
+
+  has_many :daily_calculations, :dependent => :destroy   
+  
+  # Required hook to perform the calculation associated with the Root Calculation
+  def perform_calculation
+    update_attributes(:computed_value => rand())
+  end
+  
+  # Required hook to create and queue the dependent work
+  def create_and_enqueue_dependent_calculations
+    (0...7).map do |n|
+      self.daily_calculations.create!(:day => n)
+    end.each(&:add_to_queue)
+  end
+```
+
+```ruby
+class DailyCalculation < ActiveRecord::Base
+  include QueueProcessor::DependentCalculationGroup
+  dependent_calculation_group_config do
+    self.dependent_calculation_association = :hourly_calculations
+    self.parent_calculation = :weekly_calculation
+    self.priority = lambda {|daily_calculation| 1}
+  end
+  
+  belongs_to :weekly_calculation
+  has_many :hourly_calculations, :dependent => :destroy
+    
+  # Required hook to create and queue dependent work, partitioned to this group
+  def create_and_enqueue_dependent_calculations
+    (0...23).map do |n|
+      self.hourly_calculations.create!(:hour => n)
+    end.each(&:add_to_queue)
+  end
+  
+  # Call back invoked when a calculation finished and any old calculations for this day should be cleaned up
+  def cleanup_old_calculations
+    begin
+      self.with_lock do
+        query = DailyCalculation.where(:weekly_calculation_id => self.parent_calculation.id, :day => self.day).where("id < ?", self.id)
+        query.destroy_all
+      end
+    rescue ActiveRecord::RecordNotFound => e
+    end
+  end
+end
+```
+
+```ruby
+class HourlyCalculation < ActiveRecord::Base
+  include QueueProcessor::DependentCalculation
+  dependent_calculation_config do
+    self.parent_calculation = :daily_calculation
+    self.priority = lambda {|daily_calculation| 2}
+  end
+
+  belongs_to :daily_calculation
+
+  # Required hook that computes the hourly value
+  def perform_calculation
+    update_attributes(:computed_value => rand())
+  end
+end
+```
+
+

data/Rakefile

@@ -0,0 +1,38 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'QueueProcessor'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task :default => :test

data/lib/queue-processor/config_base.rb

@@ -0,0 +1,27 @@
+module QueueProcessor
+  module ConfigBase
+    extend ActiveSupport::Concern
+    
+    included do |base|
+      base.class_eval do
+        class_attribute :queue_processor_config
+        self.queue_processor_config ||= {}
+        extend Config
+      end
+    end  
+    
+    module Config
+      private
+      def rw_config(key, value, default_value = nil, read_value = nil)
+       if value == read_value
+         queue_processor_config.include?(key) ? queue_processor_config[key] : default_value
+       else
+         config = queue_processor_config.clone
+         config[key] = value
+         self.queue_processor_config = config
+         value
+       end
+      end
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation/config.rb

@@ -0,0 +1,56 @@
+module QueueProcessor
+  module DependentCalculation
+    module Config
+      extend ActiveSupport::Concern
+      include QueueProcessor::ConfigBase
+      include QueueProcessor::QueueControlConfig
+
+      included do
+        extend Config
+      end
+
+      module Config
+
+        def dependent_calculation_config(&block)
+          yield self if block_given?
+        end
+
+        # The name of the belongs_to that is our parent
+        #
+        # * <tt>Default:</tt> :dependent_calculation_group
+        # * <tt>Accepts:</tt> Symbol
+        def parent_calculation(value = nil)
+          rw_config(:parent_calculation, value, :dependent_calculation_group)
+        end
+        alias_method :parent_calculation=, :parent_calculation
+
+        # When the calculation should run.
+        #
+        # * <tt>Default:</tt> A proc evaluating to nil (now in DJ terms)
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def run_at(value = nil)
+          rw_config(:run_at, value, lambda {|object| nil })
+        end
+        alias_method :run_at=, :run_at
+
+        # Priority of the calculation
+        #
+        # * <tt>Default:</tt> A proc delegating priority to the parent
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def priority(value = nil)
+          rw_config(:priority, value, lambda {|object| object.parent_calculation&.priority })
+        end
+        alias_method :priority=, :priority
+
+        # Which named queue should be used
+        #
+        # * <tt>Default:</tt> A proc delegating queue to the parent (if applicable) nil (any queu)
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def queue(value = nil)
+          rw_config(:queue, value, lambda { |object| object.parent_calculation&.queue })
+        end
+        alias_method :queue=, :queue
+      end
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation/queue_control.rb

@@ -0,0 +1,56 @@
+module QueueProcessor
+  module DependentCalculation
+    module QueueControl
+
+      extend ActiveSupport::Concern
+
+      def describe
+        "DependentCalculation: #{self.id}"
+      end
+
+      def delayed_jobs
+        Delayed::Backend::ActiveRecord::Job.where("handler ilike '%QueueProcessor::DependentCalculation::QueueControl::CalculationRequest%' and handler like '%id: #{id}%'")
+      end
+
+      def process_queued_calculation
+        unless parent_calculation && parent_calculation.should_cancel?
+          self.perform_calculation
+        end
+        self.update_attributes(:completed => true)
+        Rails.logger.debug {"#{self.describe}: is done"}
+
+        parent_calculation&.fire_events(:finish)
+      end
+
+      def add_to_queue
+        update_attribute(:queued_at, Time.now.utc)
+        Delayed::Job.enqueue(CalculationRequest.new(id, queued_at, describe, self.class), delayed_job_options)
+      end
+
+      def in_queue?
+        delayed_jobs.count > 0
+      end
+
+      def delayed_job_options
+         {:priority=>priority, :run_at => run_at, :queue => queue}.reject {|k,v| v.nil?}
+      end
+
+      class CalculationRequest < Struct.new(:id, :queued_at, :comment, :clazz)
+        def perform
+          obj = clazz.where(:id => id).first
+          if obj.present?
+              obj.process_queued_calculation
+          else
+            Rails.logger.warn("#{clazz.to_s} #{id}: does not exist")
+          end
+        end
+
+        def display_name
+          comment
+        end
+
+      end
+
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation/relationships.rb

@@ -0,0 +1,15 @@
+module QueueProcessor
+  module DependentCalculation
+    module Relationships
+      extend ActiveSupport::Concern
+      
+      included do
+        def parent_calculation
+          self.send( self.class.parent_calculation )
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/queue-processor/dependent_calculation.rb

@@ -0,0 +1,13 @@
+require 'active_support/concern'
+
+module QueueProcessor
+  module DependentCalculation
+    extend ActiveSupport::Concern
+
+    include QueueProcessor::DependentCalculation::Config
+    include QueueProcessor::DependentCalculation::QueueControl
+    include QueueProcessor::DependentCalculation::Relationships
+  
+  
+  end
+end

data/lib/queue-processor/dependent_calculation_group/config.rb

@@ -0,0 +1,65 @@
+module QueueProcessor
+  module DependentCalculationGroup
+    module Config
+      extend ActiveSupport::Concern
+      include QueueProcessor::ConfigBase
+      include QueueProcessor::QueueControlConfig
+
+      included do
+        extend Config
+      end
+
+      module Config
+        def dependent_calculation_group_config(&block)
+         yield self if block_given?
+        end
+
+        # The name of the has_many association giving our calculations
+        #
+        # * <tt>Default:</tt> :dependent_calculations
+        # * <tt>Accepts:</tt> Symbol
+        def dependent_calculation_association(value = nil)
+         rw_config(:dependent_calculation_association, value, :dependent_calculations)
+        end
+        alias_method :dependent_calculation_association=, :dependent_calculation_association
+
+        # The name of the belongs_to association giving the root_calculation
+        #
+        # * <tt>Default:</tt> :root_calculation
+        # * <tt>Accepts:</tt> Symbol
+        def parent_calculation(value = nil)
+         rw_config(:parent_calculation, value, :root_calculation)
+        end
+        alias_method :parent_calculation=, :parent_calculation
+
+        # When the calculation should run.
+        #
+        # * <tt>Default:</tt> A proc evaluating to nil (now in DJ terms)
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def run_at(value = nil)
+          rw_config(:run_at, value, lambda {|object| nil })
+        end
+        alias_method :run_at=, :run_at
+
+        # Priority of the calculation
+        #
+        # * <tt>Default:</tt> A proc evaluating to 1
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def priority(value = nil)
+          rw_config(:priority, value, lambda {|object| 1 })
+        end
+        alias_method :priority=, :priority
+
+
+        # Which named queue should be used
+        #
+        # * <tt>Default:</tt> A proc delegating queue to the parent (if applicable) nil (any queu)
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def queue(value = nil)
+          rw_config(:queue, value, lambda { |object| object.parent_calculation&.queue})
+        end
+        alias_method :queue=, :queue
+      end
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation_group/queue_control.rb

@@ -0,0 +1,55 @@
+module QueueProcessor
+  module DependentCalculationGroup
+    module QueueControl
+      extend ActiveSupport::Concern
+
+      def should_cancel?
+        parent_calculation.nil? || parent_calculation.calculation_stale?
+      end
+
+      def describe
+        "#{self.class.to_s} - queue dependencies"
+      end
+
+      def queue_work
+        Delayed::Job.enqueue(CalculationRequest.new(id, describe, self.class), delayed_job_options)
+      end
+
+      def delayed_jobs
+        Delayed::Backend::ActiveRecord::Job.where("handler ilike '%DependentCalculationGroup::QueueControl::CalculationRequest%' and handler like '%id: #{id}%'")
+      end
+
+      def something_queued?
+        in_queue? || dependencies_in_queue?
+      end
+
+      def in_queue?
+        delayed_jobs.count > 0
+      end
+
+      def dependencies_in_queue?
+        dependent_calculations.inject(false) { |result, dependent_calculation| result || dependent_calculation.in_queue? }
+      end
+
+      def delayed_job_options
+         {:priority=>priority, :run_at => run_at, :queue => queue}.reject {|k,v| v.nil?}
+      end
+
+      class CalculationRequest < Struct.new(:id, :comment, :clazz)
+        def perform
+          obj = clazz.where(:id => id).first
+          if obj.present?
+            obj.fire_events(:process)
+          else
+            Rails.logger.warn("#{clazz} #{id}: does not exist")
+          end
+        end
+
+        def display_name
+          "queue dependent calculations for #{comment}"
+        end
+      end
+
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation_group/relationships.rb

@@ -0,0 +1,23 @@
+module QueueProcessor
+  module DependentCalculationGroup
+    module Relationships
+      extend ActiveSupport::Concern
+      
+      included do
+        def parent_calculation
+          self.send( self.class.parent_calculation )
+        end
+
+        def dependent_calculations
+          self.send( self.class.dependent_calculation_association )
+        end
+
+        def dependent_calculations_done?
+          dependent_calculations.count == dependent_calculations.where(:completed => true).count
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/queue-processor/dependent_calculation_group/state_machines.rb

@@ -0,0 +1,77 @@
+module QueueProcessor
+  module DependentCalculationGroup
+    module StateMachines
+      extend ActiveSupport::Concern
+     
+      # optional calculation associated with this group
+      def perform_calculation
+      end
+      
+      included do
+        # this state machine is used to keep track of a group of related calculations that were created from a parent calculation
+        # When the dependent calculations all finish, then the root calculation should be marked as done.  However, if the parent calculation 
+        # become stale (by being queued agian) then we should cancel our calculations - they are no longer valid and release the
+        # parent calculation to be calculated again
+        state_machine :state, :initial => :initial, :use_transactions => false do
+          state :initial, :queued, :processing, :waiting, :complete, :cancelled, :old do
+          end
+
+          event :add_to_queue do
+            transition :initial => :queued
+          end
+
+          event :process do
+            transition :queued => :cancelled, :if => lambda {|me| me.should_cancel? }
+            transition :queued => :processing
+          end
+
+          # we don't fire this event, this is just to show you that in cleanup, we expire old calculations
+          event :expire do
+            transition [:cancelled, :complete] => :old
+          end
+
+          event :finish do
+            transition :processing => :complete, :if =>  lambda {|me| me.dependent_calculations.empty? || me.dependent_calculations_done?}
+            transition :processing => :waiting, :if => lambda {|me| !me.dependent_calculations.empty? }
+            transition [:processing, :waiting] => :cancelled, :if => lambda {|me| me.should_cancel? && me.dependent_calculations_done? }
+            transition [:processing, :waiting] => :complete,  :if  =>lambda {|me| me.dependent_calculations_done?}
+            transition :waiting => :waiting # still more work to do
+          end
+
+          # Put ourselves on the queue
+          after_transition :initial => :queued do |me|
+            Rails.logger.debug {"#{me.describe}: is being added to the queue"}
+            
+            me.update_attribute(:queued_at, Time.now.utc)
+            me.queue_work
+          end    
+          
+          # We're off the queue, get to work
+          after_transition :queued => :processing do |me|
+            Rails.logger.debug {"#{me.describe}: is being processed"}           
+
+            me.perform_calculation
+            me.create_and_enqueue_dependent_calculations
+            me.fire_events(:finish)
+          end
+
+          # We are about to finish, cleanup old calculations and perform any
+          # finalization with these calculations.  For instance, rank optimizing
+          # them.
+          before_transition any => [:complete] do |me|
+            me.cleanup_old_calculations
+          end
+
+          # Finished - clean up
+          after_transition any => [:complete, :cancelled] do |me|
+            Rails.logger.debug {"#{me.describe}: is done (#{me.cancelled? ? 'Cancelled' : 'Complete'})"}
+            me.parent_calculation.fire_events(:finish_dependent_calculations)
+          end
+
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/queue-processor/dependent_calculation_group.rb

@@ -0,0 +1,13 @@
+require 'active_support/concern'
+
+module QueueProcessor
+  
+  module DependentCalculationGroup
+    extend ActiveSupport::Concern
+    
+    include QueueProcessor::DependentCalculationGroup::Config
+    include QueueProcessor::DependentCalculationGroup::StateMachines
+    include QueueProcessor::DependentCalculationGroup::QueueControl
+    include QueueProcessor::DependentCalculationGroup::Relationships
+  end
+end

data/lib/queue-processor/queue_control_config.rb

@@ -0,0 +1,28 @@
+module QueueProcessor
+  module QueueControlConfig
+    extend ActiveSupport::Concern
+
+    def run_at
+      _config_method(:run_at)
+    end
+
+    def priority
+      _config_method(:priority)
+    end
+
+    def queue
+      _config_method(:queue)
+    end
+
+    def _config_method(method)
+      # call the method on the class, this should either give us a proc, a string or something else
+      result = self.class.send(method)
+      return nil if result.nil?
+
+      # call if it a method, send if it we respond to it as a method call, otherwise return the result
+      result.respond_to?(:call) ? result.call(self) :
+        self.respond_to?(result.to_s.to_sym) ? self.send(result.to_s.to_sym) : result
+    end
+
+  end
+end

data/lib/queue-processor/root_calculation/config.rb

@@ -0,0 +1,70 @@
+module QueueProcessor
+  module RootCalculation
+    module Config
+      extend ActiveSupport::Concern
+      include QueueProcessor::ConfigBase
+      include QueueProcessor::QueueControlConfig
+
+      included do
+        extend Config
+      end
+
+      def recover_after
+        self.class.recover_after
+      end
+
+      module Config
+
+        def root_calculation_config(&block)
+          yield self if block_given?
+        end
+
+        # The name of the has_many association are our dependent calculations
+        #
+        # * <tt>Default:</tt> :dependent_calculation_groups
+        # * <tt>Accepts:</tt> Symbol
+        def dependent_calculation_group_association(value = nil)
+          rw_config(:dependent_calculation_group_association, value, :dependent_calculation_groups)
+        end
+        alias_method :dependent_calculation_group_association=, :dependent_calculation_group_association
+
+        # When the calculation should run.
+        #
+        # * <tt>Default:</tt> A proc evaluating to 5 seconds in the future
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def run_at(value = nil)
+          rw_config(:run_at, value, lambda {|object| Time.now + 5.seconds })
+        end
+        alias_method :run_at=, :run_at
+
+        # Priority of the calculation
+        #
+        # * <tt>Default:</tt> A proc evaluating to 0 (highest priority in DJ terms)
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def priority(value = nil)
+          rw_config(:priority, value, lambda {|object| 0 })
+        end
+        alias_method :priority=, :priority
+
+        # How long we'll let a job set in the queue before process it anyway
+        #
+        # * <tt>Default:</tt> nil
+        # * <tt>Accepts:</tt> a time, for example: 10.minutes
+        def recover_after(value = nil)
+          rw_config(:recover_after, value, nil )
+        end
+        alias_method :recover_after=, :recover_after
+
+
+        # Which named queue should be used
+        #
+        # * <tt>Default:</tt> nil, any queue
+        # * <tt>Accepts:</tt> Proc or Symbol
+        def queue(value = nil)
+          rw_config(:queue, value, nil)
+        end
+        alias_method :queue=, :queue
+      end
+    end
+  end
+end

data/lib/queue-processor/root_calculation/queue_control.rb

@@ -0,0 +1,145 @@
+module QueueProcessor
+  module RootCalculation
+    module QueueControl
+
+      extend ActiveSupport::Concern
+
+
+      def before_enqueue
+      end
+
+      mattr_accessor :delayed_job_queued_at
+
+      def delayed_jobs
+        Delayed::Backend::ActiveRecord::Job.where("handler ilike '%RootCalculation::QueueControl::CalculationRequest%' and handler like '%id: #{id}%'")
+      end
+
+      def add_to_queue
+        old_queue_time = self.queued_at
+        queue_time = Time.now.utc
+
+        # We used to round(3) but that was sometimes too aggressive.  There seems
+        # to either be a float precision error, or there is a difference in the
+        # seralization of Time and TimeStamp records via delayed job / postgres
+        # that would very rarely cause round(3) comparisions to fail when they
+        # should have succeeded
+        if (old_queue_time.to_f.round(2) == queue_time.to_f.round(2))
+          Rails.logger.warn("#{self.describe}: queued again too quickly")
+          sleep(0.001)
+          add_to_queue
+          return
+        end
+
+        Rails.logger.info("#{self.describe}: added to the queue at #{queue_time} (#{queue_time.to_f})")
+
+        before_enqueue
+        touch
+
+        # XXX BOGON ALERT DO NOT USE UPDATE_ATTRIBUTE HERE - rails will do a second-based comparision of the timestamp
+        # and will not update the column if only the milliseconds are different (https://gist.github.com/johnnaegle/0599240ca925f445f031)
+        update_column(:queued_at, queue_time)
+
+        do_expire_calculation
+
+        self.queue_work(queue_time)
+      end
+
+      # expire our GMI - make it stale
+      def do_expire_calculation
+        begin
+          self.with_lock do
+            fire_events!(:expire_calculation)
+          end
+        rescue StateMachine::Error => e
+          Rails.logger.warn("#{self.describe}: invalid state transition expiring calculation: #{self.attributes}, changed: #{self.changed_attributes.map {|k,v| {k => {v => self.send(k) }}}}")
+          raise e
+        end
+      end
+
+      # called when the delayed job is deserailized and ready to run.  The calculation
+      # can be performed if the time the delayed job was queued matches the time
+      # the calculation was queued.  We round the times to two places as there
+      # seem to be float / serialization differences between postgres / YAML /
+      # Time / TimeStamp objects.  We didn't quite identify the problem, but
+      # this fixes error - jobs not running when they should.
+      def can_start?
+        self.queued_at.present? && self.queued_at.to_f.round(2) == self.delayed_job_queued_at.to_f.round(2)
+      end
+
+      def describe
+        "Calculate #{self.class.to_s}: #{self.id}"
+      end
+
+      def queue_work(queued_at)
+        Rails.logger.debug {"#{self.describe}: queued at #{queued_at.to_f}"}
+        Delayed::Job.enqueue(CalculationRequest.new(self.id, queued_at, self.describe, self.class), delayed_job_options)
+      end
+
+      def delayed_job_options
+         {:priority=>priority, :run_at => run_at, :queue => queue}.reject {|k,v| v.nil?}
+      end
+
+
+      def requeue_work(queued_at)
+        Rails.logger.debug {"#{self.describe}: re-queued at #{queued_at.to_f}"}
+        do_expire_calculation # ensure that our calculation in statel - we're in the queue, it better be
+        queue_work(queued_at)
+      end
+
+      def something_queued?
+        in_queue? || dependencies_in_queue?
+      end
+
+      def in_queue?
+        delayed_jobs.count > 0
+      end
+
+      def dependencies_in_queue?
+        dependent_calculation_groups.inject(false) { |result, dependent_calculation_group| result|| dependent_calculation_group.something_queued? }
+      end
+
+      class CalculationRequest < Struct.new(:id, :queued_at, :description, :clazz)
+        def perform
+          obj = clazz.where("id=? and date_trunc('milliseconds',queued_at)=date_trunc('milliseconds',?::timestamp)",id, queued_at).first
+
+          if (obj.present?)
+            Rails.logger.debug {"#{obj.describe}: dequeued by delayed job"}
+
+            obj.delayed_job_queued_at = queued_at
+            if obj.fire_events(:dequeue_processing)
+              Rails.logger.info("#{obj.describe}: starting processing.")
+              result = obj.fire_events(:start_processing)
+
+              if (result)
+                Rails.logger.info("#{obj.describe}: finished calculation and done in delayed job.")
+              end
+            else
+              # somebody else is running, we have to wait for them to finish
+              if (obj.calculating?)
+                if obj.recover_after.present? && (Time.now - queued_at) >= obj.recover_after
+                  Rails.logger.error("#{obj.describe}: has been stuck processing for too long (#{obj.recover_after} seconds) - reseting state machines and re-queueing")
+                  obj.reset_state_machines!
+                  obj.add_to_queue
+                else
+                  obj.requeue_work(queued_at)
+                end
+              end
+            end
+          else
+            obj = clazz.where(:id => id).first
+            if obj.nil?
+              Rails.logger.warn("#{clazz}: #{id}: Skipping: #{id} - Does not exist")
+            else
+              Rails.logger.warn("#{obj.describe}: Skipped - Queued at: #{obj.queued_at} doesn't match delayed job queued at: #{queued_at}")
+            end
+          end
+        end
+
+        def display_name
+          "calculate #{description} (id=#{id}, queued_at=#{queued_at})"
+        end
+      end
+
+    end
+  end
+end

data/lib/queue-processor/root_calculation/relationships.rb

@@ -0,0 +1,24 @@
+module QueueProcessor
+  module RootCalculation
+    module Relationships
+
+      extend ActiveSupport::Concern
+      
+      included do
+        def current_dependent_calculations
+          dependent_calculation_groups.where("created_at >= coalesce(?,?::timestamp without time zone)", self.processing_at, Time.now.utc)
+        end
+
+        def completed_dependent_calculations
+          dependent_calculation_groups.where("state = ?", 'complete')        
+        end
+
+        def dependent_calculation_groups
+          self.send( self.class.dependent_calculation_group_association )
+        end      
+      end
+      
+    end
+  end
+end
+

data/lib/queue-processor/root_calculation/state_machines.rb

@@ -0,0 +1,225 @@
+module QueueProcessor
+  module RootCalculation
+    module StateMachines
+      extend ActiveSupport::Concern
+    
+      def reset_state_machines!
+        reload
+        update_attributes(:processing_at => nil,:calculation_state => 'available', :dependent_calculations_state => 'idle', :processing_state => 'idle')    
+      end
+      
+      def stuck?
+        (processing_waiting? || dependent_calculations_waiting?) && !dependencies_in_queue?
+      end
+      
+      def finished?
+        processing_idle? && 
+        calculation_available? && 
+        dependent_calculations_idle? &&
+        !something_queued?
+      end
+            
+      
+      # We've come off the queue and need to be calculated
+      def perform_work
+        do_calculation
+        check_and_start_dependencies 
+        publish_calculation
+
+        true
+      end
+
+      # Perform whatever calculation is associated with this class
+      def do_calculation
+        Rails.logger.debug {"#{self.describe}: starting calculation"}
+
+        # mark as waiting
+        self.with_lock do
+          
+          begin
+            self.fire_events!(:start_calculation) 
+          rescue StateMachine::Error => e
+            Rails.logger.warn("#{self.describe} invalid state transition starting calculation: #{self.attributes}, changed: #{self.changed_attributes.map {|k,v| {k => {v => self.send(k) }}}}")
+            raise e
+          end
+        end    
+
+        # callback to do the actual calculation
+        perform_calculation
+
+        # mark as calculated if we are still waiting (does this need to do a with_lock?)
+        self.with_lock do
+          self.fire_events(:finish_calculation) 
+        end
+
+        Rails.logger.debug {"#{self.describe}: finished calculation"}
+      end
+
+      # Publish the calculation we just finished.  The calculation is 
+      # published before we've finished.  We're finished when we've queued any and all dependent
+      # calculations, but the number we just computed is available
+      def publish_calculation
+        Rails.logger.debug {"#{self.describe}: publishing calculation"}
+        
+        # we can only transition from calulating => available.  If something else 
+        # queued this calculation again, we'll keep the calculation unavailable
+        self.with_lock do
+          self.fire_events(:release_calculation)
+        end
+      end
+
+      # we've finished our calculation and all dependent calculations have finished
+      def finish_work
+        Rails.logger.debug {"#{self.describe}: Cleaning up"}
+        
+        unless(processing_at.nil?)
+          elapsed_time = (Time.now - self.processing_at)
+          Rails.logger.info("#{self.describe}: finished after #{elapsed_time.round(2)} seconds")
+          self.started_processing_at = self.processing_at # so observers can record how long it took to finish
+          update_attribute(:processing_at, nil)
+        end
+        reload
+      end
+
+      # After we finish our calculation (but not all the dependencies) look for dependent work to
+      # do and queue it.  Set processing at to indicate that we have work in progress and later
+      # calculations should wait until the dependent calculations we created all finish.
+      def check_and_start_dependencies      
+        if (has_dependent_calculations?)
+          do_dependencies = self.with_lock do 
+            if (self.processing_at.nil? && !calculation_stale?)
+              update_attribute(:processing_at, delayed_job_queued_at)
+
+              # this performs a callback, create_and_enqueue_dependent_calculations, to do queue up any calculations
+              begin
+                self.fire_events!(:start_dependent_calculations)    
+              rescue StateMachine::Error => e
+                Rails.logger.warn("#{self.describe}: invalid state transition for start_dependent_calculations: #{self.attributes}, changed: #{self.changed_attributes.map {|k,v| {k => {v => self.send(k) }}}}")
+                raise e
+              end
+              
+            else
+              Rails.logger.warn("#{self.describe}: should have started dependencies, but something else set processing at ")
+              false
+            end        
+          end
+        end
+      end
+
+
+      # the queued_at stored with the delayed job.  We can go from :idle => :waiting if this matches the queued_at time on the index run
+      mattr_accessor :started_processing_at
+
+      included do
+
+        state_machine :processing_state, :initial => :idle, :use_transactions => false, :namespace => 'processing' do
+          state :idle do
+            def calculating?
+              false
+            end
+          end
+
+          state :started, :waiting do
+            def calculating?
+              true
+            end
+          end
+  
+          after_transition :idle => :started, :do => :perform_work
+  
+          before_transition [:started, :waiting] => :idle, :do => :finish_work
+
+          # an index run can be queued any time.  They can be processed if there is nothing later in the queue
+          event :dequeue do
+            transition :idle => :started, :if => lambda {|me| me.can_start?}
+          end
+  
+          event :start do
+            transition :started => :waiting, :if => lambda {|me| me.has_dependent_calculations? && me.current_dependent_calculations.present?}
+            transition :started => :idle, :if => lambda {|me| !me.has_dependent_calculations? || me.current_dependent_calculations.empty?}
+          end
+
+          event :finish do
+            transition :started => :idle, :if => lambda {|me| me.current_dependent_calculations.empty?}
+            transition :waiting => :idle
+          end
+        end
+
+
+        state_machine :calculation_state, :initial => :stale, :use_transactions => true, :namespace => 'calculation' do
+          state :available do
+            def calculation_available?
+              true
+            end
+          end
+
+          state :stale, :waiting, :calculated do
+            def calculation_available?
+              false
+            end
+          end      
+
+          # expire the GMI whenever we queue this index run
+          event :expire do
+            transition any => :stale
+          end
+
+          event :start do
+            transition [:stale, :available] => :waiting
+          end
+
+          event :finish do
+            transition :waiting => :calculated
+          end
+
+          event :release do
+            transition :calculated => :available
+          end    
+        end
+
+
+        state_machine :dependent_calculations_state, :initial => :idle, :use_transactions => false, :namespace => 'dependent_calculations' do
+          state :idle, :started, :waiting do
+          end
+
+          event :start do
+            transition :idle => :idle, :if => lambda {|me| 
+              me.calculation_stale? 
+            } # something else queued so just abort early
+            transition :idle => :started # otherwise, go ahead and queue up the work 
+          end
+  
+          # queue up the work
+          before_transition :idle => :started do |me|
+            me.create_and_enqueue_dependent_calculations
+          end
+  
+          after_transition :idle => :started do |me|
+            me.fire_events(:finish_dependent_calculations) # maybe we didn't queue anything or we finished it REALLY fast, so check that
+          end
+  
+          event :finish do
+            # TODO: can finish if there are no calculations to perform or all the calculations are completed
+            transition :waiting => :idle, :if => lambda {|me| me.current_dependent_calculations.count == me.current_dependent_calculations.where(:state => [:complete, :cancelled, :old]).count  }
+    
+            transition :started => :idle, :if => lambda {|me| !me.current_dependent_calculations.exists? }
+            transition :started => :idle, :if => lambda {|me| me.current_dependent_calculations.where(:state => [:complete, :cancelled, :old]).count == me.current_dependent_calculations.count}
+            transition :started => :waiting
+          end
+          
+          after_transition [:started, :waiting] => :idle do |me|
+            Rails.logger.info("#{me.describe} is complete.")
+            begin
+              me.fire_events!(:finish_processing)
+            rescue StateMachine::Error => e
+              Rails.logger.warn("#{me.describe} invalid state transition finishing processing: #{me.attributes}, changed: #{me.changed_attributes.map {|k,v| {k => {v => me.send(k) }}}}")
+              raise e
+            end
+            
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/queue-processor/root_calculation.rb

@@ -0,0 +1,13 @@
+require 'active_support/concern'
+
+module QueueProcessor
+  module RootCalculation
+    extend ActiveSupport::Concern
+    
+    include QueueProcessor::RootCalculation::Config
+    include QueueProcessor::RootCalculation::StateMachines
+    include QueueProcessor::RootCalculation::QueueControl
+    include QueueProcessor::RootCalculation::Relationships
+      
+  end
+end

data/lib/queue-processor/version.rb

@@ -0,0 +1,3 @@
+module QueueProcessor
+  VERSION = "0.0.8"
+end

data/lib/queue-processor.rb

@@ -0,0 +1,19 @@
+require 'queue-processor/config_base'
+require 'queue-processor/queue_control_config'
+require 'queue-processor/root_calculation/config'
+require 'queue-processor/root_calculation/queue_control'
+require 'queue-processor/root_calculation/relationships'
+require 'queue-processor/root_calculation/state_machines'
+require 'queue-processor/root_calculation'
+require 'queue-processor/dependent_calculation_group/config'
+require 'queue-processor/dependent_calculation_group/queue_control'
+require 'queue-processor/dependent_calculation_group/relationships'
+require 'queue-processor/dependent_calculation_group/state_machines'
+require 'queue-processor/dependent_calculation_group'
+require 'queue-processor/dependent_calculation/config'
+require 'queue-processor/dependent_calculation/queue_control'
+require 'queue-processor/dependent_calculation/relationships'
+require 'queue-processor/dependent_calculation'
+
+module QueueProcessor
+end

data/lib/tasks/queue-processor_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :queue-processor do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: queue-processor
+version: !ruby/object:Gem::Version
+  version: 0.0.8
+platform: ruby
+authors:
+- John Naegle
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.4
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Queue a main calculation, then groups of dependent calculations and wait
+  for them to finish.
+email:
+- john.naegle@goodmeasures.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/queue-processor.rb
+- lib/queue-processor/config_base.rb
+- lib/queue-processor/dependent_calculation.rb
+- lib/queue-processor/dependent_calculation/config.rb
+- lib/queue-processor/dependent_calculation/queue_control.rb
+- lib/queue-processor/dependent_calculation/relationships.rb
+- lib/queue-processor/dependent_calculation_group.rb
+- lib/queue-processor/dependent_calculation_group/config.rb
+- lib/queue-processor/dependent_calculation_group/queue_control.rb
+- lib/queue-processor/dependent_calculation_group/relationships.rb
+- lib/queue-processor/dependent_calculation_group/state_machines.rb
+- lib/queue-processor/queue_control_config.rb
+- lib/queue-processor/root_calculation.rb
+- lib/queue-processor/root_calculation/config.rb
+- lib/queue-processor/root_calculation/queue_control.rb
+- lib/queue-processor/root_calculation/relationships.rb
+- lib/queue-processor/root_calculation/state_machines.rb
+- lib/queue-processor/version.rb
+- lib/tasks/queue-processor_tasks.rake
+homepage: https://github.com/GoodMeasuresLLC/queue-processor
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: A processing model for a main calculation and a set of dependent calculations.
+test_files: []