pampa_dispatcher 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 854adb6af15723ba6c9a2f4ac5af8acc8a47f4ba
+  data.tar.gz: 11091e316eab1a12bfb30114269ea612693aa59a
+SHA512:
+  metadata.gz: a89cd5510431a901f9677bd35cceabcf20d281e0d514f1063cc83c514335a6998ae14109ea15078e455d61853ac63358bbc0ee86a7a637b5fe4faff5bc81ab82
+  data.tar.gz: 324445697c97efa5ed2089e43326a9363165a7d7db044929d07eacf9834cbf3c2526bfb56d7b77e1475196f058c350d92aa6cabab8ac895c3250b23b8e7a3bbc

data/lib/pampa_dispatcher.rb

@@ -0,0 +1,219 @@
+require 'pampa_workers'
+
+module BlackStack
+  
+  class Dispatcher    
+    attr_accessor :name
+    # database information
+    # :field_times, :field_start_time and :field_end_time maybe nil
+    attr_accessor :table
+    attr_accessor :field_primary_key
+    attr_accessor :field_id
+    attr_accessor :field_time 
+    attr_accessor :field_times
+    attr_accessor :field_start_time
+    attr_accessor :field_end_time
+    # max number of records assigned to a worker that have not started (:start_time field is nil)
+    attr_accessor :queue_size 
+    # max number of minutes that a job should take to process. if :end_time keep nil x minutes 
+    # after :start_time, that's considered as the job has failed or interrumped
+    attr_accessor :max_job_duration_minutes  
+    # max number of times that a record can start to process & fail (:start_time field is not nil, 
+    # but :end_time field is still nil after :max_job_duration_minutes)
+    attr_accessor :max_try_times
+    # additional function to decide how many records are pending for processing
+    # it should returns an integer
+    # keep it nil if you want to run the default function
+    attr_accessor :ocuppied_function
+    # additional function to decide if the worker can dispatch or not
+    # example: use this function when you want to decide based on the remaining credits of the client
+    # it should returns true or false
+    # keep it nil if you want it returns always true
+    attr_accessor :allowing_function
+    # additional function to choose the records to launch
+    # it should returns an array of IDs
+    # keep this parameter nil if you want to use the default algorithm
+    attr_accessor :selecting_function
+    # additional function to choose the records to retry
+    # keep this parameter nil if you want to use the default algorithm
+    attr_accessor :relaunching_function
+    # additional function to perform the update on a record to retry
+    # keep this parameter nil if you want to use the default algorithm
+    attr_accessor :relauncher_function
+    # additional function to perform the update on a record to flag the starting of the job
+    # by default this function will set the :field_start_time field with the current datetime, and it will increase the :field_times counter 
+    # keep this parameter nil if you want to use the default algorithm
+    attr_accessor :starter_function
+    # additional function to perform the update on a record to flag the finishing of the job
+    # by default this function will set the :field_end_time field with the current datetime 
+    # keep this parameter nil if you want to use the default algorithm
+    attr_accessor :finisher_function
+
+    
+    # setup dispatcher configuration here
+    def initialize(h)
+      self.name = h[:name]
+      self.table = h[:table]
+      self.field_primary_key = h[:field_primary_key]
+      self.field_id = h[:field_id]
+      self.field_time = h[:field_time]
+      self.field_times = h[:field_times]
+      self.field_start_time = h[:field_start_time]
+      self.field_end_time = h[:field_end_time]
+      self.queue_size = h[:queue_size]
+      self.max_job_duration_minutes = h[:max_job_duration_minutes]  
+      self.max_try_times = h[:max_try_times]
+      self.ocuppied_function = h[:ocuppied_function]
+      self.allowing_function = h[:allowing_function]
+      self.selecting_function = h[:selecting_function]
+      self.relaunching_function = h[:relaunching_function]
+      self.relauncher_function = h[:relauncher_function]
+    end
+    
+    # decide how many records are pending for processing
+    # it will count the number if records with :reservation_id == worker.id, and :start_time == nil
+    # it returns an integer
+    def ocuppied_slots(worker)
+      if self.ocuppied_function.nil?
+        return self.table.where(self.field_id.to_sym => worker.id, self.field_start_time.to_sym => nil).count if !self.field_start_time.nil?
+        return self.table.where(self.field_id.to_sym => worker.id).count if self.field_start_time.nil?
+      else
+        # TODO: validar que retorna un entero
+        return self.ocuppied_function.call(worker, self)
+      end
+    end
+
+    # returns the number of free slots in the procesing queue of this worker
+    def available_slots(worker)
+      ocuppied = self.ocuppied_slots(worker)
+      allowed = self.queue_size
+      if ocuppied > allowed
+        return 0
+      else
+        return allowed - ocuppied
+      end
+    end
+
+    # decide if the worker can dispatch or not
+    # example: use this function when you want to decide based on the remaining credits of the client
+    # returns always true
+    def allowing(worker)
+      if self.allowing_function.nil?
+        return true
+      else
+        # TODO: validar que retorna true o false
+        return self.allowing_function.call(worker, self)
+      end
+    end
+
+    # choose the records to dispatch
+    # returns an array of IDs
+    def selecting_dataset(worker, n)
+      ds = self.table.select(self.field_primary_key.to_sym).where(self.field_id.to_sym => nil) 
+      ds = ds.filter(self.field_end_time.to_sym => nil) if !self.field_end_time.nil?  
+      ds = ds.filter("#{self.field_times.to_s} IS NULL OR #{self.field_times.to_s} < #{self.max_try_times.to_s}") if !self.field_times.nil? 
+      ds.limit(n)
+    end # selecting_dataset
+
+    def selecting(worker, n)
+      if self.allowing_function.nil?
+        return self.selecting_dataset(worker, n).map { |o| o[self.field_primary_key.to_sym] }
+      else
+        # TODO: validar que retorna un array de strings
+        return self.selecting_function.call(worker, self, n)
+      end
+    end
+
+    # choose the records to retry
+    # returns an array of IDs
+    def relaunching_dataset(worker, n)
+      ds = self.table.select(self.field_primary_key.to_sym).where("#{self.field_time.to_s} < '#{(Time.now - 60*self.max_job_duration_minutes.to_i).strftime('%Y-%m-%d %H:%M:%S').to_s}'")
+      ds = ds.filter("#{self.field_end_time.to_s} IS NULL") if !self.field_end_time.nil?  
+#      ds = ds.filter("( #{self.field_times.to_s} IS NULL OR #{self.field_times.to_s} < #{self.max_try_times.to_s} ) ") if !self.field_times.nil?
+      ds = ds.limit(n)
+    end
+
+    def relaunching(worker, n)
+      if self.relaunching_function.nil?
+        return self.relaunching_dataset(worker, n).map { |o| o[self.field_primary_key.to_sym] }
+      else
+        # TODO: validar que retorna un array de strings
+        return self.relaunching_function.call(worker, self, n)
+      end
+    end
+    
+    def relaunch(o)
+      o[self.field_id.to_sym] = nil
+      o[self.field_time.to_sym] = nil
+      o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
+      o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
+      o.save      
+    end
+
+    def start(o)
+      if self.starter_function.nil?
+        o[self.field_start_time.to_sym] = now() if !self.field_start_time.nil?
+        o[self.field_times.to_sym] = o[self.field_times.to_sym].to_i + 1 if !self.field_times.nil?
+        o.save
+      else
+        self.starter_function.call(o, self)
+      end
+    end
+
+    def finish(o)
+      if self.finisher_function.nil?
+        o[self.field_end_time.to_sym] = now() if !self.field_end_time.nil?
+        o.save
+      else
+        self.finisher_function.call(o, self)
+      end
+    end
+    
+    # relaunch records
+    def run_relaunch(worker)
+      # relaunch failed records
+      self.relaunching(worker, self.queue_size).each { |id|
+        o = self.table.where(self.field_primary_key.to_sym => id).first
+        if self.relauncher_function.nil?
+          self.relaunch(o)
+        else
+          self.relauncher_function.call(o, self)
+        end
+        # release resources
+        DB.disconnect
+        GC.start
+      }
+    end # def run_relaunch
+    
+    # dispatch records
+    # returns the # of records dispatched
+    def run_dispatch(worker)
+      # get # of available slots
+      n = self.available_slots(worker)
+      
+      # dispatching n pending records
+      i = 0
+      if n>0
+        self.selecting(worker, n).each { |id|
+          # count the # of dispatched
+          i += 1
+          # dispatch records
+          o = self.table.where(self.field_primary_key.to_sym => id).first
+          o[self.field_id.to_sym] = worker.id
+          o[self.field_time.to_sym] = now()
+          o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
+          o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
+          o.save
+          # release resources
+          DB.disconnect
+          GC.start        
+        }
+      end
+      
+      #      
+      return i
+    end
+    
+  end # class Dispatcher
+  
+end # module BlackStack

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: pampa_dispatcher
+version: !ruby/object:Gem::Version
+  version: 1.1.1
+platform: ruby
+authors:
+- Leandro Daniel Sardi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: tiny_tds
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+- !ruby/object:Gem::Dependency
+  name: pampa_workers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/pampa_dispatcher.'
+email: leandro.sardi@expandedventure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pampa_dispatcher.rb
+homepage: https://rubygems.org/gems/pampa_dispatcher
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Distribute work along a pool of Pampa
+  workers.
+test_files: []