autoscaler 0.0.1

data/lib/autoscaler.rb

@@ -0,0 +1,5 @@
+require "autoscaler/version"
+
+# Namespace module; no code
+module Autoscaler
+end

data/lib/autoscaler/heroku_scaler.rb

@@ -0,0 +1,55 @@
+require 'heroku-api'
+
+module Autoscaler
+  # Wraps the Heroku API to provide just the interface that we need for scaling.
+  class HerokuScaler
+    # @param [String] type process type this scaler controls
+    # @param [String] key Heroku API key
+    # @param [String] app Heroku app name
+    def initialize(
+        type = 'worker',
+        key = ENV['HERKOU_API_KEY'],
+        app = ENV['HEROKU_APP'])
+      @client = Heroku::API.new(:api_key => key)
+      @type = type
+      @app = app
+      @workers = 0
+      @known = Time.now - 1
+    end
+
+    attr_reader :app
+    attr_reader :type
+
+    # Read the current worker count (value may be cached)
+    # @return [Numeric] number of workers
+    def workers
+      if known?
+        @workers
+      else
+        know client.get_ps(app).body.count {|ps| ps['process'].match /#{type}\.\d?/ }
+      end
+    end
+
+    # Set the number of workers (noop if workers the same)
+    # @param [Numeric] n number of workers
+    def workers=(n)
+      if n != @workers || !known?
+        p "Scaling #{type} to #{n}"
+        client.post_ps_scale(app, type, n)
+        know n
+      end
+    end
+
+    private
+    attr_reader :client
+
+    def know(n)
+      @known = Time.now + 5
+      @workers = n
+    end
+
+    def known?
+      Time.now < @known
+    end
+  end
+end

data/lib/autoscaler/sidekiq.rb

@@ -0,0 +1,86 @@
+require 'securerandom' # bug in Sidekiq as of 2.2.1
+require 'sidekiq'
+
+module Autoscaler
+  # namespace module for Sidekiq middlewares
+  module Sidekiq
+    # Sidekiq client middleware
+    # Performs scale-up when items are queued and there are no workers running
+    class Client
+      # @param [Hash] scalers map of queue(String) => scaler (e.g. {HerokuScaler}).
+      #   Which scaler to use for each sidekiq queue
+      def initialize(scalers)
+        @scalers = scalers
+      end
+
+      # Sidekiq middleware api method
+      def call(worker_class, item, queue)
+        @scalers[queue] && @scalers[queue].workers = 1
+        yield
+      end
+    end
+
+    # Sidekiq server middleware
+    # Performs scale-down when the queue is empty
+    class Server
+      # @param [scaler] scaler object that actually performs scaling operations (e.g. {HerokuScaler})
+      # @param [Numeric] timeout number of seconds to wait before shutdown
+      # @param [Array[String]] specified_queues list of queues to monitor to determine if there is work left.  Defaults to all sidekiq queues.
+      def initialize(scaler, timeout, specified_queues = nil)
+        @scaler = scaler
+        @timeout = timeout
+        @specified_queues = specified_queues
+      end
+
+      # Sidekiq middleware api entry point
+      def call(worker, msg, queue)
+        working!
+        yield
+      ensure
+        working!
+        wait_for_task_or_scale
+      end
+
+      private
+      def queues
+        @specified_queues || registered_queues
+      end
+
+      def registered_queues
+        ::Sidekiq.redis { |x| x.smembers('queues') }
+      end
+
+      def empty?(name)
+        ::Sidekiq.redis { |conn| conn.llen("queue:#{name}") == 0 }
+      end
+
+      def pending_work?
+        queues.any? {|q| !empty?(q)}
+      end
+
+      def wait_for_task_or_scale
+        loop do
+          return if pending_work?
+          return @scaler.workers = 0 if idle?
+          sleep(0.5)
+        end
+      end
+
+      def working!
+        ::Sidekiq.redis {|c| c.set('background_activity', Time.now)}
+      end
+
+      def idle_time
+        ::Sidekiq.redis {|c|
+          t = c.get('background_activity')
+          return 0 unless t
+          Time.now - Time.parse(t)
+        }
+      end
+
+      def idle?
+        idle_time > @timeout
+      end
+    end
+  end
+end

data/lib/autoscaler/version.rb

@@ -0,0 +1,4 @@
+module Autoscaler
+  # version number
+  VERSION = "0.0.1"
+end

data/spec/autoscaler/heroku_scaler_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'autoscaler/heroku_scaler'
+
+describe Autoscaler::HerokuScaler, :online => true do
+  let(:cut) {Autoscaler::HerokuScaler}
+  let(:client) {cut.new}
+  subject {client}
+
+  its(:workers) {should == 0}
+
+  describe 'scaled' do
+    around do |example|
+      client.workers = 1
+      example.yield
+      client.workers = 0
+    end
+
+    its(:workers) {should == 1}
+  end
+end

data/spec/autoscaler/sidekiq_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'autoscaler/sidekiq'
+
+class Scaler
+  attr_accessor :workers
+
+  def initialize(n = 0)
+    self.workers = n
+  end
+end
+
+describe Autoscaler::Sidekiq do
+  let(:scaler) do
+    Scaler.new(workers)
+  end
+
+  describe Autoscaler::Sidekiq::Client do
+    let(:cut) {Autoscaler::Sidekiq::Client}
+    let(:sa) {cut.new('queue' => scaler)}
+    let(:workers) {0}
+
+    describe 'scales' do
+      before {sa.call(Class, {}, 'queue') {}}
+      subject {scaler.workers}
+      it {should == 1}
+    end
+
+    describe 'yields' do
+      it {sa.call(Class, {}, 'queue') {:foo}.should == :foo}
+    end
+  end
+
+  describe Autoscaler::Sidekiq::Server do
+    let(:cut) {Autoscaler::Sidekiq::Server}
+    let(:sa) {cut.new(scaler, 0)}
+    let(:workers) {1}
+
+    describe 'scales' do
+      before{sa.call(Object.new, {}, 'queue') {}}
+      subject {scaler.workers}
+      it {should == 0}
+    end
+
+    describe 'yields' do
+      it {sa.call(Object.new, {}, 'queue') {:foo}.should == :foo}
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+RSpec.configure do |config|
+  config.mock_with :rspec
+
+  config.filter_run_excluding :online => true unless ENV['HEROKU_APP']
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: autoscaler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Justin Love
+- Fix Peña
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+- !ruby/object:Gem::Dependency
+  name: heroku-api
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mast
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Currently provides a Sidekiq middleware that does 0/1 scaling of Heroku
+  processes
+email:
+- git@JustinLove.name
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/autoscaler/heroku_scaler.rb
+- lib/autoscaler/sidekiq.rb
+- lib/autoscaler/version.rb
+- lib/autoscaler.rb
+- spec/autoscaler/heroku_scaler_spec.rb
+- spec/autoscaler/sidekiq_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: autoscaler
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Start/stop Sidekiq workers on Heroku
+test_files:
+- spec/autoscaler/heroku_scaler_spec.rb
+- spec/autoscaler/sidekiq_spec.rb
+- spec/spec_helper.rb
+has_rdoc: