sidekiq-heroku-scaler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 652123c97be79746c8c9266fb67547fc6c7878406a6dd313ed9de147473f6c93
+  data.tar.gz: 4e0a7e9eb222d13ba63a20ddad66675c89dffcf31dd21b5c373206628a63e6db
+SHA512:
+  metadata.gz: 607de29a191e9bd2ece43741fc606733d0d8da6f493745da965d05f61189e5bce4eaacb8b73eb3abce482475311c5bad403147cecc2c5b4d7038353649c5bcc3
+  data.tar.gz: 6c82485a0013553f3f31434d0efa9e0e4a6ab28f3de08977fbf8673791361e6f2a2767367220e42a2db9adab070e5698016081deee130911d3962ac678253fdb

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+.idea
+.ruby-gemset
+.ruby-version

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  TargetRubyVersion: 2.5
+  Exclude:
+    - 'bin/*'
+Documentation:
+  Enabled: false
+Metrics/LineLength:
+  Max: 120

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.2

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    sidekiq-heroku-scaler (0.1.0)
+      platform-api (~> 2.2)
+      sidekiq (~> 5.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    connection_pool (2.2.2)
+    diff-lcs (1.3)
+    erubis (2.7.0)
+    excon (0.62.0)
+    heroics (0.0.25)
+      erubis (~> 2.0)
+      excon
+      moneta
+      multi_json (>= 1.9.2)
+    jaro_winkler (1.5.1)
+    moneta (1.0.0)
+    multi_json (1.13.1)
+    parallel (1.12.1)
+    parser (2.5.3.0)
+      ast (~> 2.4.0)
+    platform-api (2.2.0)
+      heroics (~> 0.0.25)
+      moneta (~> 1.0.0)
+    powerpack (0.1.2)
+    rack (2.0.6)
+    rack-protection (2.0.5)
+      rack
+    rainbow (3.0.0)
+    rake (12.3.2)
+    redis (4.1.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.61.1)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.4.0)
+    ruby-progressbar (1.10.0)
+    sidekiq (5.2.3)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    unicode-display_width (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  rake
+  rspec
+  rubocop
+  sidekiq-heroku-scaler!
+
+BUNDLED WITH
+   1.17.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Vital Ryabchinskiy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+# SidekiqHerokuScaler
+
+Tool to autoscale sidekiq dynos on Heroku. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-heroku-scaler'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-heroku-scaler
+
+## Usage
+
+### Strategy
+
+Create strategy instance (gem strategy based on latency):
+```
+scale_strategy = SidekiqHerokuScaler::Strategy::Latency.new(
+  min_dynos_count: 1,
+  max_dynos_count: 10,
+  max_latency: 5.minutes.to_i,
+  min_latency: 1.minute.to_i
+)
+```
+or
+
+define your own strategy:
+```
+class CustomStrategy
+  def increase?(sidekiq_worker)
+    # TODO
+  end 
+	
+  def decrease?(sidekiq_worker)
+    # TODO
+  end
+end
+```
+
+methods `increase?/decrease?` are required, these methods provide logic to handle additing or removing sidekiq instance.
+
+### Manager
+
+To run iteration
+```
+SidekiqHerokuScaler::Manager.new(
+  heroku_app_name: HEROKU_APP_NAME,
+  heroku_token: HEROKU_TOKEN,
+  strategy: scale_strategy,
+  workers: SIDEKIQ_AUTOSCALE_WORKERS
+).perform
+```
+
+where:
+- `HEROKU_APP_NAME` - Heroku app name
+- `HEROKU_TOKEN` - Heroku token
+- `strategy` - scale strategy
+- `workers` - array of sidekiq worker names that could be scaled
+
+### Recurring
+
+1) Implement rake task and use [Heroku Scheduler](https://devcenter.heroku.com/articles/scheduler)
+2) To more frequently run use [Sidekiq Scheduler](https://github.com/moove-it/sidekiq-scheduler)
+
+###### `config/sidekiq_scheduler.yml`
+```
+sidekiq_autoscale:
+  cron: '0 */5 * * * *'
+  class: Scheduling::SidekiqAutoscaleWorker
+  queue: autoscale
+  description: 'This job autoscale sidekiq Heroku dynos'
+```
+
+###### `Scheduling::SidekiqAutoscaleWorker`
+```
+module Scheduling
+  class SidekiqAutoscaleWorker
+    include Sidekiq::Worker
+
+    sidekiq_options retry: false
+
+    def perform
+      SidekiqHerokuScaler::Manager.new(
+        heroku_app_name: ENV['HEROKU_APP_NAME'],
+        heroku_token: ENV['HEROKU_TOKEN'],
+        strategy: scale_strategy,
+        workers: %w[worker]
+      ).perform
+    end
+end    
+```
+
+## TODO
+
+- Lack of specs
+- More documentation
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sidekiq-heroku-scaler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sidekiq-heroku-scaler.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'sidekiq-heroku-scaler/strategy/latency'
+require 'sidekiq-heroku-scaler/manager'
+require 'sidekiq-heroku-scaler/version'

data/lib/sidekiq-heroku-scaler/heroku/client.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'platform-api'
+require 'sidekiq-heroku-scaler/heroku/formation'
+
+module SidekiqHerokuScaler
+  module Heroku
+    class Client
+      def initialize(heroku_app_name, heroku_token)
+        @heroku_app_name = heroku_app_name
+        @heroku_token = heroku_token
+      end
+
+      def formations
+        @formations ||= formation.list(heroku_app_name)
+      end
+
+      def formation_for(worker_name)
+        SidekiqHerokuScaler::Heroku::Formation.new(
+          formations.detect { |formation| formation['type'] == worker_name } || {}
+        )
+      end
+
+      def sidekiq_workers
+        @sidekiq_workers ||= begin
+          formations.select { |formation| formation['command'].match(/sidekiq/) }
+                    .map { |formation| formation['type'] }
+        end
+      end
+
+      def update_formation(formation_id, quantity)
+        formation.update(heroku_app_name, formation_id, quantity: quantity)
+      end
+
+      private
+
+      attr_reader :heroku_app_name, :heroku_token
+
+      def client
+        @client ||= PlatformAPI.connect_oauth(heroku_token)
+      end
+
+      def formation
+        @formation ||= PlatformAPI::Formation.new(client)
+      end
+    end
+  end
+end

data/lib/sidekiq-heroku-scaler/heroku/formation.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'ostruct'
+
+module SidekiqHerokuScaler
+  module Heroku
+    class Formation < OpenStruct
+    end
+  end
+end

data/lib/sidekiq-heroku-scaler/manager.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'sidekiq-heroku-scaler/heroku/client'
+require 'sidekiq-heroku-scaler/worker'
+
+module SidekiqHerokuScaler
+  class Manager
+    def initialize(heroku_app_name:, heroku_token:, workers:, strategy:)
+      @heroku_client = SidekiqHerokuScaler::Heroku::Client.new(heroku_app_name, heroku_token)
+      @strategy = strategy
+      @workers = workers
+    end
+
+    def perform
+      autoscalable_workers.each(&method(:autoscale_one))
+    end
+
+    private
+
+    attr_reader :heroku_client, :strategy, :workers
+
+    def autoscale_one(worker_name)
+      formation = heroku_client.formation_for(worker_name)
+      return if formation.blank?
+
+      sidekiq_worker = SidekiqHerokuScaler::Worker.new(worker_name, formation)
+
+      process_formation(sidekiq_worker)
+    end
+
+    def process_formation(sidekiq_worker)
+      if strategy.increase?(sidekiq_worker)
+        heroku_client.update_formation(sidekiq_worker.formation_id, sidekiq_worker.quantity + 1)
+      elsif strategy.decrease?(sidekiq_worker)
+        heroku_client.update_formation(sidekiq_worker.formation_id, sidekiq_worker.quantity - 1)
+      end
+    end
+
+    def autoscalable_workers
+      heroku_client.sidekiq_workers & workers
+    end
+  end
+end

data/lib/sidekiq-heroku-scaler/strategy/latency.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SidekiqHerokuScaler
+  module Strategy
+    class Latency
+      def initialize(min_dynos_count:, max_dynos_count:, max_latency:, min_latency:)
+        @min_dynos_count = min_dynos_count
+        @max_dynos_count = max_dynos_count
+        @max_latency = max_latency
+        @min_latency = min_latency
+      end
+
+      def increase?(sidekiq_worker)
+        sidekiq_worker.quantity < max_dynos_count &&
+          sidekiq_worker.latency > max_latency &&
+          sidekiq_worker.queues_size > sidekiq_worker.quantity * sidekiq_worker.concurrency
+      end
+
+      def decrease?(sidekiq_worker)
+        sidekiq_worker.latency < min_latency && sidekiq_worker.quantity > min_dynos_count
+      end
+
+      private
+
+      attr_reader :min_dynos_count, :max_dynos_count, :max_latency, :min_latency
+    end
+  end
+end

data/lib/sidekiq-heroku-scaler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SidekiqHerokuScaler
+  VERSION = '0.1.0'
+end

data/lib/sidekiq-heroku-scaler/worker.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+
+module SidekiqHerokuScaler
+  class Worker
+    def initialize(worker_name, formation)
+      @worker_name = worker_name
+      @formation = formation
+    end
+
+    def concurrency
+      process['concurrency'] || 0
+    end
+
+    def formation_id
+      formation.id
+    end
+
+    def quantity
+      formation.quantity
+    end
+
+    def latency
+      queues.sum { |queue| Sidekiq::Queue.new(queue).latency }
+    end
+
+    def queues_size
+      queues.sum { |queue| Sidekiq::Queue.new(queue).size }
+    end
+
+    private
+
+    attr_reader :formation, :worker_name
+
+    def queues
+      process['queues'] || []
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def process
+      process_set.detect { |p| p.identity.match(/\A#{worker_name}\./) } || {}
+    end
+  end
+end

data/sidekiq-heroku-scaler.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq-heroku-scaler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sidekiq-heroku-scaler'
+  spec.version       = SidekiqHerokuScaler::VERSION
+  spec.authors       = ['Vital Ryabchinskiy']
+  spec.email         = ['vital.ryabchinskiy@gmail.com']
+
+  spec.summary       = 'Sidekiq Heroku Scaler'
+  spec.description   = 'Tool to scale sidekiq dynos on Heroku'
+  spec.homepage      = 'https://github.com/vitalinfo/sidekiq-heroku-scaler'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+
+  spec.add_dependency 'platform-api', '~> 2.2'
+  spec.add_dependency 'sidekiq', '~> 5.2'
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-heroku-scaler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vital Ryabchinskiy
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: platform-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+description: Tool to scale sidekiq dynos on Heroku
+email:
+- vital.ryabchinskiy@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sidekiq-heroku-scaler.rb
+- lib/sidekiq-heroku-scaler/heroku/client.rb
+- lib/sidekiq-heroku-scaler/heroku/formation.rb
+- lib/sidekiq-heroku-scaler/manager.rb
+- lib/sidekiq-heroku-scaler/strategy/latency.rb
+- lib/sidekiq-heroku-scaler/version.rb
+- lib/sidekiq-heroku-scaler/worker.rb
+- sidekiq-heroku-scaler.gemspec
+homepage: https://github.com/vitalinfo/sidekiq-heroku-scaler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Sidekiq Heroku Scaler
+test_files: []