scale_workers 0.0.1

data/lib/auto_scale.rb

@@ -0,0 +1,108 @@
+class AutoScale
+
+  LOAD_DECREMENT_FACTOR = 2 #2x times
+
+  module LOAD_LISTENER
+    SLEEP     = 2 #seconds
+    THRESHOLD = 3
+  end
+
+  attr_accessor :queue, :stopped, :current_workers_count, :current_pid_switch, :previous_pid_switch, :previous_workers_count
+
+  attr_accessor :interrupted, :high_load
+
+  def monitor
+    bind_interrupt_listener
+    usage_listener = bind_high_usage_listener
+    monitor_workers
+  ensure
+    Thread.kill(usage_listener) if usage_listener
+  end
+
+  def monitor_workers
+    loop do
+      begin
+        if self.interrupted
+          exit
+          break
+        elsif self.high_load
+          high_load_decrement
+          scale_workers
+          self.high_load = false
+        else
+          increment_or_decrement
+        end
+      rescue Exception => e
+        self.interrupted = true
+      end
+    end
+  end
+
+  def stop(count)
+    ScaleWorkers.configuration.stop_procedure(self.queue, count)
+  end
+
+  def start(count)
+    ScaleWorkers.configuration.start_procedure(self.queue, count)
+  end
+
+  def jobs_count
+    ScaleWorkers.configuration.count_procedure(self.queue, self.max_failure)
+  end
+
+  def exit
+    stop self.current_workers_count
+  end
+
+  def increment_or_decrement
+    if jobs_count > 0
+      load = LoadMonitor.can_increase_load?(self.max_cpu_load, self.max_memory_load)    
+      self.current_workers_count += self.increment_step if(load  && self.previous_workers_count < self.max_workers)
+      self.current_workers_count -= self.decrement_step if(!load && self.previous_workers_count > self.min_workers)
+      self.current_workers_count  = 0 if self.current_workers_count < 0
+    else
+      self.current_workers_count = self.min_workers
+    end
+    scale_workers
+    sleep(self.sleep_time)
+  end
+
+  private
+  def bind_interrupt_listener
+    Signal.trap('TERM') do
+      self.interrupted = true
+    end
+
+    Signal.trap('INT') do
+      self.interrupted = true
+    end
+  end
+
+  def bind_high_usage_listener
+    Thread.new do
+      high_load_counter = 0
+      loop do
+        if LoadMonitor.cpu_load > self.max_cpu_load || LoadMonitor.memory_load > self.max_memory_load
+          high_load_counter += 1
+        else
+          high_load_counter = 0
+        end
+        self.high_load = true if high_load_counter >= LOAD_LISTENER::THRESHOLD
+        sleep(LOAD_LISTENER::SLEEP)
+      end
+    end
+  end
+
+  def scale_workers
+    return if self.current_workers_count == self.previous_workers_count
+    stop(self.previous_workers_count)
+    start(self.current_workers_count)
+    self.previous_workers_count = self.current_workers_count
+  end
+
+  def high_load_decrement
+    self.current_workers_count -= (LOAD_DECREMENT_FACTOR * self.decrement_step)
+    self.current_workers_count = 0 if self.current_workers_count < 0
+  end
+
+end

data/lib/load_monitor.rb

@@ -0,0 +1,29 @@
+module LoadMonitor
+
+  def self.vendor
+    UsageWatch
+  end
+
+  def self.cpu_load
+    vendor.uw_cpuused
+  end
+
+  def self.memory_load
+    vendor.uw_memused
+  end
+
+  def self.can_increase_load?(max_cpu_load, max_memory_load)
+    increment_count  = 0
+    self.load_cycles.times do
+      if self.cpu_load < max_cpu_load && self..memory_load < max_memory_load
+        increment_count += 1
+      else
+        increment_count -= 2
+      end
+      sleep(self.load_sleep_time)
+    end
+    # increment_count > 0 ? say("Load normal-Increment") : say("Load high-Decrement")
+    return increment_count > 0
+  end
+
+end

data/lib/scale_workers.rb

@@ -0,0 +1,21 @@
+require 'usagewatch'
+require 'active_support'
+require 'scale_workers/utils'
+require 'active_support/core_ext'
+require 'scale_workers/configuration'
+require 'scale_workers/auto_scale'
+require 'scale_workers/load_monitor'
+
+module ScaleWorkers
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configuration
+    @@configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(self.configuration)
+  end
+end

data/lib/scale_workers.task

@@ -0,0 +1,6 @@
+namespace :scale_workers do
+  desc "Start Scale workers"
+  task :monitor => :environment do
+    ScaleWorkers::AutoScale.new.monitor
+  end
+end

data/lib/scale_workers/adapter/delayed_job_active_record.rb

@@ -0,0 +1,65 @@
+module ScaleWorkers
+  module Adapter
+
+    class DelayedJobActiveRecord
+
+      include ScaleWorkers::Utils
+
+      PRIME   = 97
+      TIMEOUT = 5
+
+      module Command
+        START = 'start'
+        STOP  = 'stop'
+      end
+
+      attr_accessor :pid_switch
+
+      def initialize
+        @pid_switch = 0
+      end
+
+      def stop_procedure
+        lambda do |queue, count|
+          say('Inside stop procedure')
+          adapter = self#::ScaleWorkers.configuration.adapter
+          pid_dir = adapter.pid_dir(queue)
+          adapter.dj_call(adapter.class::Command::STOP, queue, pid_dir, count)
+          sleep(adapter.class::TIMEOUT)
+        end
+      end
+
+      def start_procedure
+        lambda do |queue, count|
+          say('Inside start procedure')
+          adapter = self #::ScaleWorkers.configuration.adapter
+          adapter.increment_pid_switch
+          pid_dir = adapter.pid_dir(queue)
+          adapter.dj_call(adapter.class::Command::START, queue, pid_dir, count)
+        end
+      end
+
+      def count_procedure
+        lambda {|queue, max_failure| Delayed::Job.where(queue: queue).where("attempts <= ?", max_failure).count}
+      end
+
+      def increment_pid_switch
+        self.pid_switch = (self.pid_switch + 1) % PRIME
+      end
+
+      def pid_dir(queue)
+        File.join(Rails.root.to_s, "tmp", "pids", "#{queue}.#{self.pid_switch}")
+      end
+
+      def dj_call(command, queue, pid_dir, count)
+        say('DJ call')
+        `cd #{Rails.root.to_s}; RAILS_ENV=#{Rails.env} #{::ScaleWorkers.configuration.worker_executable_path} --queue='#{queue}' -p'#{queue}' --pid-dir=#{pid_dir} -n#{count} #{command}`
+      end
+
+      def pid_switch
+        @pid_switch ||= 0
+      end
+    end
+
+  end
+end

data/lib/scale_workers/auto_scale.rb

@@ -0,0 +1,126 @@
+module ScaleWorkers
+  class AutoScale
+
+    include ScaleWorkers::Utils
+
+    LOAD_DECREMENT_FACTOR = 2 #2x times
+
+    module LOAD_LISTENER
+      SLEEP     = 2 #seconds
+      THRESHOLD = 3
+    end
+
+    attr_accessor :queue, :current_workers_count, :previous_workers_count
+    attr_accessor :interrupted, :high_load, :config
+
+    def initialize(queue = 'default')
+      self.queue = queue
+      self.config = ScaleWorkers.configuration
+      self.current_workers_count = 0
+      self.previous_workers_count = 0
+      self.high_load = false
+      self.interrupted = false
+    end
+
+    def monitor
+      bind_interrupt_listener
+      usage_listener = bind_high_usage_listener
+      monitor_workers
+    ensure
+      Thread.kill(usage_listener) if usage_listener
+    end
+
+    def monitor_workers
+      loop do
+        say 'monitoring'
+        begin
+          if self.interrupted
+            exit
+            break
+          elsif self.high_load
+            say('high_load')
+            high_load_decrement
+            scale_workers
+            self.high_load = false
+          else
+            increment_or_decrement
+          end
+        rescue Exception => e
+          raise e
+          self.interrupted = true
+        end
+      end
+    end
+
+    def stop(count)
+      config.stop_procedure.call(self.queue, count)
+    end
+
+    def start(count)
+      config.start_procedure.call(self.queue, count)
+    end
+
+    def jobs_count
+      config.count_procedure.call(self.queue, config.max_failure)
+    end
+
+    def exit
+      stop self.current_workers_count
+    end
+
+    def increment_or_decrement
+      if jobs_count > 0
+        load = LoadMonitor.can_increase_load?(config.max_cpu_load, config.max_memory_load)    
+        self.current_workers_count += config.increment_step if(load  && self.previous_workers_count < config.max_workers)
+        self.current_workers_count -= config.decrement_step if(!load && self.previous_workers_count > config.min_workers)
+        self.current_workers_count  = 0 if self.current_workers_count < 0
+      else
+        self.current_workers_count = config.min_workers
+      end
+      scale_workers
+      sleep(config.sleep_time)
+    end
+
+    private
+    def bind_interrupt_listener
+      Signal.trap('TERM') do
+        self.interrupted = true
+        say('Term signal received')
+      end
+
+      Signal.trap('INT') do
+        self.interrupted = true
+        say('INT signal received')
+      end
+    end
+
+    def bind_high_usage_listener
+      Thread.new do
+        high_load_counter = 0
+        loop do
+          say 'listening'
+          if ScaleWorkers::LoadMonitor.cpu_load > config.max_cpu_load || ScaleWorkers::LoadMonitor.memory_load > config.max_memory_load
+            high_load_counter += 1
+          else
+            high_load_counter = 0
+          end
+          self.high_load = true if high_load_counter >= LOAD_LISTENER::THRESHOLD
+          sleep(LOAD_LISTENER::SLEEP)
+        end
+      end
+    end
+
+    def scale_workers
+      return if self.current_workers_count == self.previous_workers_count
+      stop(self.previous_workers_count)
+      start(self.current_workers_count)
+      self.previous_workers_count = self.current_workers_count
+    end
+
+    def high_load_decrement
+      self.current_workers_count -= (LOAD_DECREMENT_FACTOR * config.decrement_step)
+      self.current_workers_count = 0 if self.current_workers_count < 0
+    end
+
+  end
+end

data/lib/scale_workers/configuration.rb

@@ -0,0 +1,46 @@
+module ScaleWorkers
+  class Configuration
+
+    attr_accessor :max_failure, :max_workers, :min_workers, :sleep_time, :notification_interval, :increment_step, :decrement_step,
+                  :max_memory_load, :max_cpu_load, :load_cycles, :load_sleep_time
+
+    attr_accessor :start_procedure, :stop_procedure, :count_procedure
+
+    attr_accessor :adapter, :worker_executable_path
+
+    def initialize
+      self.max_failure           = 5
+      self.max_workers           = 5
+      self.min_workers           = 1
+      self.sleep_time            = 5.minutes
+      self.increment_step        = 1
+      self.decrement_step        = -2
+      self.notification_interval = 15
+      # machine load attrs
+      self.max_memory_load       = 70
+      self.max_cpu_load          = 50
+      self.load_cycles           = 5
+      self.load_sleep_time       = 10
+      self.adapter               = 'delayed_job_active_record'
+      self.worker_executable_path = 'script/delayed_job'
+    end
+
+    def adapter=(adapter)
+      require "scale_workers/adapter/#{adapter}"
+      @adapter = "ScaleWorkers::Adapter::#{adapter.classify}".constantize.new
+    end
+
+    def start_procedure
+      @start_procedure.presence || self.adapter.start_procedure
+    end
+
+    def stop_procedure
+      @stop_procedure.presence || self.adapter.stop_procedure
+    end
+
+    def count_procedure
+      @count_procedure.presence || self.adapter.count_procedure
+    end
+
+  end
+end

data/lib/scale_workers/load_monitor.rb

@@ -0,0 +1,31 @@
+module ScaleWorkers
+  module LoadMonitor
+
+    def self.vendor
+      Usagewatch
+    end
+
+    def self.cpu_load
+      vendor.uw_cpuused
+    end
+
+    def self.memory_load
+      vendor.uw_memused
+    end
+
+    def self.can_increase_load?(max_cpu_load, max_memory_load)
+      increment_count  = 0
+      ScaleWorkers.configuration.load_cycles.times do
+        if self.cpu_load < max_cpu_load && self.memory_load < max_memory_load
+          increment_count += 1
+        else
+          increment_count -= 2
+        end
+        sleep(ScaleWorkers.configuration.load_sleep_time)
+      end
+      # increment_count > 0 ? say("Load normal-Increment") : say("Load high-Decrement")
+      return increment_count > 0
+    end
+
+  end
+end

data/lib/scale_workers/utils.rb

@@ -0,0 +1,7 @@
+module ScaleWorkers
+  module Utils
+    def say(message)
+      p message
+    end
+  end
+end

data/lib/scale_workers_railtie.rb

@@ -0,0 +1,5 @@
+class ScaleWorkersRailtie < Rails::Railtie
+  rake_tasks do
+    load "../Rakefile"
+  end
+end

data/scale_workers.gemspec

@@ -0,0 +1,20 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.add_dependency 'delayed_job_active_record', ['>=4.0.2']
+  spec.add_dependency 'usagewatch', ['0.0.7']
+  spec.authors        = ['Karthick Shankar']
+  spec.description    = 'Auto scaling job workers for delayed_job_active_record'
+  spec.summary        = 'Stop worrying about scaling job workers'
+  spec.email          = ['karibtech@gmail.com']
+  spec.files          = %w[scale_workers.gemspec]
+  spec.files         += Dir.glob('lib/**/*.rb')
+  spec.files         += ['lib/scale_workers.task']
+  spec.licenses       = ['MIT']
+  spec.name           = 'scale_workers'
+  spec.require_paths  = ['lib']
+  spec.version        = '0.0.1'
+  spec.add_development_dependency 'minitest', ['~> 3.1']
+  spec.add_development_dependency 'mocha', ['1.0.0']
+  spec.test_files     = Dir.glob('test/**/*.rb')
+end

data/test/load_monitor_test.rb

@@ -0,0 +1,31 @@
+require 'test_helper'
+
+class LoadMonitorTest < MiniTest::Test
+
+  def test_vendor
+    assert_equal Usagewatch, ScaleWorkers::LoadMonitor.vendor
+  end
+
+  def test_cpu_load
+    Usagewatch.expects(:uw_cpuused).once.returns(10.5)
+    assert_equal 10.5, ScaleWorkers::LoadMonitor.cpu_load
+  end
+
+  def test_memory_load
+    Usagewatch.expects(:uw_memused).once.returns(55)
+    assert_equal 55, ScaleWorkers::LoadMonitor.memory_load
+  end
+
+  def test_can_increase_load_normal_load_case
+    Usagewatch.expects(:uw_cpuused).times(5).returns(50).then.returns(51).then.returns(58).then.returns(53).then.returns(40)
+    Usagewatch.expects(:uw_cpuused).times(5).returns(69).then.returns(60).then.returns(70).then.returns(50).then.returns(2)
+    refute ScaleWorkers::LoadMonitor.can_increase_load?(55, 70)
+  end
+
+  def test_can_increase_load_high_load_case
+    Usagewatch.expects(:uw_cpuused).returns(55).times(5).then.returns(51).then.returns(58).then.returns(53).then.returns(40)
+    Usagewatch.expects(:uw_memused).returns(69).times(5).then.returns(60).then.returns(70).then.returns(80).then.returns(22)
+    assert ScaleWorkers::LoadMonitor.can_increase_load?(55, 70)
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'scale_workers'
+require 'minitest/autorun'
+require 'minitest/unit'
+require 'minitest/pride'
+require 'mocha/mini_test'

data/test/utils_test.rb

@@ -0,0 +1,14 @@
+require 'test_helper'
+
+class UtilsTest < MiniTest::Test
+
+  def setup
+    @object = Object.new
+    @object.extend(ScaleWorkers::Utils)
+  end
+
+  def test_say
+    assert_equal "Hello", @object.say("Hello")
+  end
+
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: scale_workers
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Karthick Shankar
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+- !ruby/object:Gem::Dependency
+  name: usagewatch
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.7
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Auto scaling job workers for delayed_job_active_record
+email:
+- karibtech@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- scale_workers.gemspec
+- lib/scale_workers/load_monitor.rb
+- lib/scale_workers/adapter/delayed_job_active_record.rb
+- lib/scale_workers/utils.rb
+- lib/scale_workers/auto_scale.rb
+- lib/scale_workers/configuration.rb
+- lib/scale_workers_railtie.rb
+- lib/load_monitor.rb
+- lib/scale_workers.rb
+- lib/auto_scale.rb
+- lib/scale_workers.task
+- test/utils_test.rb
+- test/load_monitor_test.rb
+- test/test_helper.rb
+homepage: 
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Stop worrying about scaling job workers
+test_files:
+- test/utils_test.rb
+- test/load_monitor_test.rb
+- test/test_helper.rb