dynomatic 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 171c4347b0535537b19ec7138c29d234401b97a4
+  data.tar.gz: aef7a01fe6eb21928d1b8c2f31c5e5ebec060c03
+SHA512:
+  metadata.gz: 0d027ba8438005b550f4c2e5d0456a1feb14bb821bc5755cfa0f4ca7247af035f48247cf14b20128315d08dff61354f41801250163b1094c81aa69f5a1ce293d
+  data.tar.gz: bd5cf41e57fb11a320abb53d23b8705c52ae30d58c015c292a02783911ee2fc441c01ac5eb9560d87c96b1e6354a1ce95c6cc8df8a2031e1651fd09a32cda9f3

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.16.0

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in dynomatic.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    dynomatic (0.1.0)
+      activejob
+      activesupport
+      platform-api
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (5.1.4)
+      activesupport (= 5.1.4)
+      globalid (>= 0.3.6)
+    activesupport (5.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    delayed_job (4.1.4)
+      activesupport (>= 3.0, < 5.2)
+    diff-lcs (1.3)
+    erubis (2.7.0)
+    excon (0.60.0)
+    globalid (0.4.1)
+      activesupport (>= 4.2.0)
+    heroics (0.0.24)
+      erubis (~> 2.0)
+      excon
+      moneta
+      multi_json (>= 1.9.2)
+    i18n (0.9.1)
+      concurrent-ruby (~> 1.0)
+    method_source (0.9.0)
+    minitest (5.11.1)
+    moneta (0.8.1)
+    multi_json (1.13.1)
+    platform-api (2.1.0)
+      heroics (~> 0.0.23)
+      moneta (~> 0.8.1)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.4)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  delayed_job
+  dynomatic!
+  pry
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 David Verhasselt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,126 @@
+# Dynomatic
+
+Use this gem to automatically scale up/down your Heroku worker dynos based on the number of jobs currently waiting in the queue. Currently only DelayedJob is supported but it's really easy to add support for your preferred background job library.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dynomatic'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dynomatic
+
+## Usage
+
+Configure it as follows:
+
+```ruby
+# In an initializer:
+Dynomatic.configure do |config|
+  # Optional, since it will try to detect the correct adapter: 
+  # config.adapter = Dynomatic::Adapters::DelayedJob
+
+  # Check your heroku account for the correct token, or run
+  #
+  #   heroku auth:token
+  #
+  config.heroku_token = ENV["HEROKU_TOKEN"]
+  config.heroku_app = ENV["HEROKU_APP_NAME"]
+
+  # Add your own rules here. Dynomatic will select the lowest dyno count that
+  # matches the "at_least" number of jobs.
+  #
+  # Some examples based on the following rules:
+  #   25 jobs = 10 dynos, 31 jobs = 15 dynos
+  config.rules = [
+    {at_least: 0, dynos: 1},
+    {at_least: 10, dynos: 5},
+    {at_least: 20, dynos: 10},
+    {at_least: 30, dynos: 15},
+  ]
+
+  # Optional (see README), let Dynomatic know which are the worker dynos:
+  # config.worker_names = %w(worker1 worker2 worker3 worker4)
+end
+```
+
+## What if I'm using Hobby dynos?
+
+Heroku only allows one to scale up/down dynos if they're the more expensive "professional" dynos. If you're using Hobby dynos, you can only start and stop them.
+
+To work around that, we can just create a bunch of different dyno types, and then enable/disable them as a means of scaling up or down.
+
+First you need to setup the dynos in your `Procfile`. Wheras before it probably looked a bit like this:
+
+```
+web: bundle exec puma -C config/puma.rb
+worker: bundle exec rails jobs:work
+```
+
+Now add a bunch of worker-types with the exact same command:
+
+```
+web: bundle exec puma -C config/puma.rb
+worker1: bundle exec rails jobs:work
+worker2: bundle exec rails jobs:work
+worker3: bundle exec rails jobs:work
+worker4: bundle exec rails jobs:work
+```
+
+You can add however many you want, but we can't scale beyond the amount that you define here.
+
+Next, in the Dynomatic configuration, add the names of those worker dyno types:
+
+```
+Dynomatic.configure do |config|
+  # ...
+  config.worker_names = %w(worker1 worker2 worker3 worker4)
+
+  # Since we only have 4 worker dyno types, we can only scale up to 4:
+  config.rules = [
+    {at_least: 0, dynos: 1},
+    {at_least: 50, dynos: 2},
+    {at_least: 100, dynos: 4},
+  ]
+end
+```
+
+Now instead of scaling up/down a singular `worker` dyno depending on the rules, Dynomatic will instead start/stop the next dyno type in the `worker_names` type.
+
+Considering the above example, if there's 55 jobs in the queue, Dynomatic will start both `worker1` and `worker2` and stop `worker3` and `worker4`, while if there's only 20, it will only start `worker1` and stop the remainder.
+
+## FAQ
+
+_Does this take into account jobs in the future, or failed, retrying jobs?_
+
+No, it only counts jobs that need to be run right now.
+
+_How do you ensure multi-thread/multi-process workers don't all try to scale the workers_
+I don't, each worker thread of each worker process will do their own call to the API. It's no problem if Heroku receives multiple calls to set the dyno-count to e.g. 20, it will ignore the duplicates. Since there's 4500 requests per hour available for the API, this should also never exhaust the API calls.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/dv/dynomatic.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+
+## TODO
+- Hysteresis (will be hard without using a distributed state storage device)
+- Other ActiveJob adapters beyond DelayedJob

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "dynomatic"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dynomatic.gemspec

@@ -0,0 +1,33 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "dynomatic/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "dynomatic"
+  spec.version       = Dynomatic::VERSION
+  spec.authors       = ["David Verhasselt"]
+  spec.email         = ["david@crowdway.com"]
+
+  spec.summary       = %q{Gem to autoscale Heroku worker dynos based on number of pending jobs}
+  spec.description   = %q{Gem to autoscale Heroku worker dynos based on number of pending jobs}
+  spec.homepage      = "https://github.com/dv/dynomatic"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activejob"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "platform-api"
+
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "delayed_job"
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/dynomatic.rb

@@ -0,0 +1,22 @@
+require "active_job"
+require "dynomatic/version"
+
+module Dynomatic
+  extend ActiveSupport::Autoload
+
+  autoload :Adapters
+  autoload :Configuration
+  autoload :Master
+  autoload :Scaler
+  autoload :HobbyScaler
+
+  mattr_accessor(:configuration) { Configuration.new }
+
+  def self.configure
+    configuration.adapter ||= Dynomatic::Adapters.detect
+
+    yield(configuration)
+
+    Dynomatic::Master.new(configuration).install!
+  end
+end

data/lib/dynomatic/adapters.rb

@@ -0,0 +1,18 @@
+module Dynomatic
+  module Adapters
+    extend ActiveSupport::Autoload
+
+    autoload :DelayedJob
+
+    module_function
+
+    def detect
+      case
+      when defined? ::Delayed::Job
+        Dynomatic::Adapters::DelayedJob.new
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/dynomatic/adapters/delayed_job.rb

@@ -0,0 +1,9 @@
+module Dynomatic
+  module Adapters
+    class DelayedJob
+      def job_count
+        Delayed::Job.where("run_at <= NOW()").count
+      end
+    end
+  end
+end

data/lib/dynomatic/configuration.rb

@@ -0,0 +1,9 @@
+module Dynomatic
+  class Configuration
+    attr_accessor :adapter, :rules, :worker_names, :heroku_token, :heroku_app
+
+    def sorted_rules
+      @rules.sort_by { |rule| rule[:over] }
+    end
+  end
+end

data/lib/dynomatic/hobby_scaler.rb

@@ -0,0 +1,23 @@
+module Dynomatic
+  class HobbyScaler
+    attr :scaler, :worker_processes
+
+    def initialize(heroku_token, heroku_app, worker_processes)
+      @scaler = Scaler.new(heroku_token, heroku_app)
+      @worker_processes = worker_processes
+    end
+
+    def scale_to(dyno_count)
+      start_processes = Array(worker_processes.first(dyno_count))
+      stop_processes = Array(worker_processes[dyno_count..-1])
+
+      start_processes.each do |process|
+        scaler.start!(process)
+      end
+
+      stop_processes.each do |process|
+        scaler.stop!(process)
+      end
+    end
+  end
+end

data/lib/dynomatic/master.rb

@@ -0,0 +1,56 @@
+module Dynomatic
+  class Master
+    attr_accessor :current_dyno_count, :configuration
+
+    def initialize(configuration)
+      @configuration = configuration
+    end
+
+    def install!
+      this = self
+
+      ActiveJob::Base.before_perform do
+        this.adjust_dynos!
+      end
+    end
+
+    # Main method
+    def adjust_dynos!
+      job_count = configuration.adapter.job_count
+      new_dyno_count = dyno_count_for_job_count(job_count)
+
+      if current_dyno_count != new_dyno_count
+        set_dyno_count!(new_dyno_count)
+      end
+    end
+
+    private
+
+    def dyno_count_for_job_count(job_count)
+      rule = configuration.sorted_rules.reverse.find { |rule| rule[:at_least] <= job_count }
+
+      rule[:dynos]
+    end
+
+    def set_dyno_count!(new_dyno_count)
+      old_dyno_count = current_dyno_count
+      self.current_dyno_count = new_dyno_count
+
+      scaler.scale_to(new_dyno_count)
+    rescue => error
+      # If something went wrong, reset to old value
+      self.current_dyno_count = old_dyno_count
+
+      Rails.logger.error "Something went wrong while adjusting dyno count: #{error.message}."
+    end
+
+    def scaler
+      @scaler ||=
+        if configuration.worker_names.present?
+          HobbyScaler.new(configuration.heroku_token, configuration.heroku_app, configuration.worker_names)
+        else
+          Scaler.new(configuration.heroku_token, configuration.heroku_app)
+        end
+    end
+  end
+end

data/lib/dynomatic/scaler.rb

@@ -0,0 +1,32 @@
+require "platform-api"
+
+module Dynomatic
+  class Scaler
+    attr :heroku_token, :heroku_app
+
+    def initialize(heroku_token, heroku_app)
+      @heroku_token = heroku_token
+      @heroku_app = heroku_app
+    end
+
+    def scale_to(dyno_count, type: "worker")
+      Rails.logger.info "Scaling #{type} to #{dyno_count}"
+
+      client.formation.update(heroku_app, type, {quantity: dyno_count})
+    end
+
+    def start!(type)
+      scale_to(1, type: type)
+    end
+
+    def stop!(type)
+      scale_to(0, type: type)
+    end
+
+    private
+
+    def client
+      @client = ::PlatformAPI.connect_oauth(heroku_token)
+    end
+  end
+end

data/lib/dynomatic/version.rb

@@ -0,0 +1,3 @@
+module Dynomatic
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,175 @@
+--- !ruby/object:Gem::Specification
+name: dynomatic
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- David Verhasselt
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: platform-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Gem to autoscale Heroku worker dynos based on number of pending jobs
+email:
+- david@crowdway.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dynomatic.gemspec
+- lib/dynomatic.rb
+- lib/dynomatic/adapters.rb
+- lib/dynomatic/adapters/delayed_job.rb
+- lib/dynomatic/configuration.rb
+- lib/dynomatic/hobby_scaler.rb
+- lib/dynomatic/master.rb
+- lib/dynomatic/scaler.rb
+- lib/dynomatic/version.rb
+homepage: https://github.com/dv/dynomatic
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Gem to autoscale Heroku worker dynos based on number of pending jobs
+test_files: []