flynn_auto_scale 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 57177c265ec52b7370f5435554bbf8ea95389504
+  data.tar.gz: 5315b450481d61a0a38c82840c0f9107455c66d3
+SHA512:
+  metadata.gz: 01a8da9664023ff21b36392cd14f21e17701f242fb6d110f0c83d4b1ffaafbdc8d3ac476b00e28aa7ebd9d40c0f59b04463d56d23cd7f22102b2cd1f18700d4c
+  data.tar.gz: d7a0eecf24e8e5b4a5786c06c1d0e293b8f79f7a34f2e40f86ef7cdb7b45901b6cad8c7e67b6f77e4f6d41869ee331b81e124772fed7c56a8cee74b8d4dfa5a4

data/README.md

@@ -0,0 +1,58 @@
+Warning: This gem is in active development and may have some serious bugs.
+
+# FlynnAutoScale
+A gem that allows your Rails apps to self-scale as they need more and more resources under [Flynn](https://flynn.io/). Comes with an automated mode that can be used for "day to day" web hosting operations and a manual mode where you can control when scaling operations take place (for scripts, background jobs and anything else)
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'flynn_auto_scale'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Make sure to migrate after installing this gem (it makes use of ActiveRecord to keep the autoscaler working)
+
+In order to get up and running quickly, please set up the following ENV variables.
+Failure to set up some of these could cause "bad things to happen", so ensure you read this section.
+
+    # These are the minimum required items for [connecting to a cluster](https://flynn.io/docs/cli#adding-clusters)
+    FLYNN_SETUP_CLUSTER_PIN
+    FLYNN_SETUP_CLUSTER_NAME
+    FLYNN_SETUP_CLUSTER_CONTROLLER_DOMAIN
+    FLYNN_SETUP_CLUSTER_CONTROLLER_KEY
+    FLYNN_SETUP_CLUSTER_APP_NAME - the app name to be scaled.
+
+
+    # These are configuration variables that are used to customize how FlynnAutoScale will handle scaling events.
+    FLYNN_AUTO_SCALE - If present, FlynnAutoScale will attempt auto scaling operations -> (default: do nothing)
+    FLYNN_AUTO_SCALE_RAM - The amount of RAM in MB you'd like FlynnAutoScale to consider scaling at -> (default: 256)
+    FLYNN_AUTO_SCALE_COOLDOWN - How long to wait after you have scaled (in seconds) before scaling again -> (default: 0)
+    FLYNN_LIMIT_INSTANCES_MANUAL_MODE - If present, whether manual mode should follow the instance limits -> (default: ignore limits)
+    FLYNN_MIN_INSTANCES - Minimum number of instances -> (default: 1)
+    FLYNN_MAX_INSTANCES - Maximum number of instances -> (default: 2)
+    
+Add the following at the top of your application_controller.rb to get started with auto scaling.
+
+```ruby
+# This should be a single line
+around_action def scale; yield; FlynnAutoScale::Scaler.auto_scale('web'); end
+```
+
+## Usage
+For a more advanced use of the scaler, you can directly call the manual scaler yourself.
+```ruby
+# You can pass the process name: 'console', 'web', 'worker'.
+FlynnAutoScale::Scaler.scale_manual('web', 2)
+```
+## Contributing
+It's an open source repo, fork the thing, make some changes, and I'll put you on the list of cool people making Flynn + Rails great again.
+
+I am also looking for a partner in crime or two to turn that database of scaling changes into a nice dashboard (a la sidekiq's web_ui). I can do it, but it's gonna be super butt ugly if I do it alone.
+
+## License
+The gem is available as open source under the CC0 terms.

data/Rakefile

@@ -0,0 +1,37 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'FlynnAutoScale'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+load 'rails/tasks/statistics.rake'
+
+
+
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/app/assets/config/flynn_auto_scale_manifest.js

@@ -0,0 +1,2 @@
+//= link_directory ../javascripts/flynn_auto_scale .js
+//= link_directory ../stylesheets/flynn_auto_scale .css

data/app/assets/javascripts/flynn_auto_scale/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/flynn_auto_scale/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/flynn_auto_scale/application_controller.rb

@@ -0,0 +1,5 @@
+module FlynnAutoScale
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/helpers/flynn_auto_scale/application_helper.rb

@@ -0,0 +1,4 @@
+module FlynnAutoScale
+  module ApplicationHelper
+  end
+end

data/app/jobs/flynn_auto_scale/application_job.rb

@@ -0,0 +1,4 @@
+module FlynnAutoScale
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/mailers/flynn_auto_scale/application_mailer.rb

@@ -0,0 +1,6 @@
+module FlynnAutoScale
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/flynn_auto_scale/application_record.rb

@@ -0,0 +1,5 @@
+module FlynnAutoScale
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/flynn_auto_scale/scaling_event.rb

@@ -0,0 +1,4 @@
+module FlynnAutoScale
+  class ScalingEvent < ApplicationRecord
+  end
+end

data/app/views/layouts/flynn_auto_scale/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Flynn auto scale</title>
+  <%= stylesheet_link_tag    "flynn_auto_scale/application", media: "all" %>
+  <%= javascript_include_tag "flynn_auto_scale/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/initializers/flynn_auto_scale.rb

@@ -0,0 +1,3 @@
+# This initializer will attempt to connect to the cluster as soon as the app is brought online.
+# The internals of the gem should prevent errors but if you get an error, just comment out this line.
+FlynnAutoScale::Scaler.connect_cluster

data/config/routes.rb

@@ -0,0 +1,2 @@
+FlynnAutoScale::Engine.routes.draw do
+end

data/db/migrate/20170530115155_create_flynn_auto_scale_scaling_events.rb

@@ -0,0 +1,10 @@
+class CreateFlynnAutoScaleScalingEvents < ActiveRecord::Migration[5.0]
+  def change
+    create_table :flynn_auto_scale_scaling_events do |t|
+      t.string :event_type
+      t.integer :instances
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20170530121037_add_process_type_to_scaling_event.rb

@@ -0,0 +1,5 @@
+class AddProcessTypeToScalingEvent < ActiveRecord::Migration[5.0]
+  def change
+    add_column :scaling_events, :process_type, :string
+  end
+end

data/lib/flynn_auto_scale.rb

@@ -0,0 +1,196 @@
+require "flynn_auto_scale/engine"
+require "os"
+
+module FlynnAutoScale
+  class Scaler < Rails::Engine
+    # START Setup Section
+    def check_install
+      # Check for existence of Flynn inside container.
+      output = `flynn`
+      if output.include? "usage: flynn"
+        true
+      else
+        false
+      end
+    end
+    
+    def install
+      # Install the Flynn CLI if it doesn't exist.
+      if !check_install
+        Logger.warn "Flynn Auto-Scale: Installing the Flynn CLI. This will take some time. This installer only works under Linux / Mac OS X."
+        `L=/usr/local/bin/flynn && curl -sSL -A "\`uname -sp\`" https://dl.flynn.io/cli | zcat >$L && chmod +x $L`
+        check_install
+      else
+        true
+      end
+    end
+    
+    def connect_cluster
+      # Connect to the cluster and set as default.
+      if !install
+        # Warn the user something has gone terribly wrong.
+        Logger.warn "Flynn Auto-Scale: There was an issue installing / verifying the Flynn installation. Auto Scaling functions will not work."
+        false
+      else
+        # The bare minimum needed to connect to a cluster and manual scaling
+        vars = [
+        ENV['FLYNN_SETUP_CLUSTER_PIN'],
+        ENV['FLYNN_SETUP_CLUSTER_NAME'],
+        ENV['FLYNN_SETUP_CLUSTER_CONTROLLER_DOMAIN'],
+        ENV['FLYNN_SETUP_CLUSTER_CONTROLLER_KEY'],
+        ENV['FLYNN_SETUP_CLUSTER_APP_NAME']]
+        
+        # Verify the existance of all of the required variables.
+        missing_vars = false
+        vars.each do |var|
+          if !var.present?
+            Logger.warn "Flynn Auto-Scale: One of the environment variables required for connecting to the cluster is missing. Initialization will stop immediately."
+            missing_vars = true
+          end
+        end
+        
+        # Abort if there are missing variables.
+        if missing_vars
+          return false
+        end
+        
+        # Connect to the cluster.
+        # Warning: this method of adding / defaulting the cluster is susceptible to Command Injection.
+        # Do not pass params from the internet into any of this (you were warned!)
+        # Source: http://brakemanscanner.org/docs/warning_types/command_injection/
+        `flynn cluster add -p #{vars[0]} #{vars[1]} #{vars[2]} #{vars[3]}`
+        
+        output = `flynn cluster default #{vars[4]}`
+        if !output.include? "is now the default cluster."
+          Logger.warn "Flynn Auto-Scale: There was an issue connecting to the cluster. Please provide this output in a Github Issue: #{output}"
+          false
+        else
+          true
+        end
+      end
+    end
+    # END Setup Section
+    
+    # START Auto Scaling Section
+    def auto_scale(process='web')
+      # This method will utilize the current RAM usage and some ENV variables to make a scaling decision.
+      if !ENV['FLYNN_AUTO_SCALE'].present?
+        Logger.warn "Flynn Auto-Scale: A call was made to the auto_scale method but the FLYNN_AUTO_SCALE ENV variable is not set. Nothing will be done."
+        return
+      end
+      
+      current_ram_use = (OS.rss_bytes / 1024).to_i
+      max_ram_use = ENV['FLYNN_AUTO_SCALE_RAM'].present? ? ENV['FLYNN_AUTO_SCALE_RAM'].present?.to_i : 256
+      
+      if current_ram_use > max_ram_use
+        # Consider scaling upwards.
+        scale_up(process)
+      elsif current_ram_use < (max_ram_use * 0.1).to_i
+        # This is a hardcoded attempt at a "downscale".
+        # We will attempt to shrink the instance count if using less than 10% of the max ram.
+        scale_down(process)
+      else
+        # There is no need to execute a scale up / down event.
+        return
+      end
+    end
+    # END Auto Scaling Section
+    
+    # START Maintenance Section
+    def can_scale_time(process='web')
+      # Checks if scaling is appropriate or if it's limited by a time minimum.
+      last_scale = FlynnAutoScale::ScalingEvent.where(process_type: process).last
+      scaling_restriction = ENV['FLYNN_AUTO_SCALE_COOLDOWN'].present?
+      
+      if scaling_restriction
+        if !last_scale
+          # If this is the first time this process is scaled, just ignore the limit and push an initial scaling event into the DB.
+          last_scale = FlynnAutoScale::ScalingEvent.create(process_type: process, event_type: 'initial', instances: 1)
+          true
+        else
+          # Calculate if we have breached the cooldown.
+          current_time = DateTime.now
+          last_time = last_scale.created_at
+          diff = (current_time.to_f - last_time.to_f).to_i
+          
+          # Scaling operations should occur if the cooldown has elapsed.
+          if diff > ENV['FLYNN_AUTO_SCALE_COOLDOWN'].to_i
+            true
+          else
+            false
+          end
+        end
+      else
+        # No limit, just scale.
+        true
+      end
+    end
+    def scale_up(process='web')
+      # This method will scale the process count up. It assumes it was called from an "AUTO_SCALE" setup.
+      if can_scale_time
+        scale(process, FlynnAutoScale::ScalingEvent.where(process_type: process).last.instances + 1, 'scale_up')
+      else
+        Logger.warn "Flynn Auto-Scale: A scale_up event was aborted due to a time limitation."
+      end
+    end
+    
+    def scale_down(process='web')
+      # This method will scale the process count down. It assumes it was called from an "AUTO_SCALE" setup.
+      if can_scale_time
+        scale(process, FlynnAutoScale::ScalingEvent.where(process_type: process).last.instances - 1, 'scale_down')
+      else
+        Logger.warn "Flynn Auto-Scale: A scale_down event was aborted due to a time limitation."
+      end
+    end
+    
+    def scale_manual(process='web', instances=1)
+      # This method will just scale the process. 
+      # This method will respect very little restrictions, make sure you know what this does.
+      scale(process, instances)
+    end
+    
+    # The foundation of the auto / manual scaling system.
+    # You should not be calling this method unless you know what you are doing.
+    def scale(process='web', instances=1, event_type='manual')
+      if event_type == 'manual' && !ENV['FLYNN_LIMIT_INSTANCES_MANUAL_MODE'].present?
+        # If the instances are not limited in manual mode, just let it rip (doesn't wait for confirmation)
+        FlynnAutoScale::ScalingEvent.create(process_type: process, event_type: event_type, instances: instances)
+        `flynn -a #{ENV['FLYNN_SETUP_CLUSTER_APP_NAME']} scale #{process}=#{instances} -n`
+      else
+        # This section is active in auto mode and in manual mode with limitations.
+        if ENV['FLYNN_MIN_INSTANCES'].present? && ENV['FLYNN_MAX_INSTANCES'].present? 
+          if instances < ENV['FLYNN_MIN_INSTANCES'].to_i
+            Logger.warn "Flynn Auto-Scale: A scale event was stopped due to being below the minimum instance count"
+            return
+          end
+          
+          # Do the min and max instance checks
+            
+          if instances > ENV['FLYNN_MAX_INSTANCES'].to_i
+            Logger.warn "Flynn Auto-Scale: A scale event was stopped due to being above the maximum instance count"
+            return
+          end
+        elsif ENV['FLYNN_MIN_INSTANCES'].present?
+          # Do the min instance check
+          if instances < ENV['FLYNN_MIN_INSTANCES'].to_i
+            Logger.warn "Flynn Auto-Scale: A scale event was stopped due to being below the minimum instance count"
+            return
+          end
+        elsif ENV['FLYNN_MAX_INSTANCES'].present?
+          # Do the max instance check
+          if instances > ENV['FLYNN_MAX_INSTANCES'].to_i
+            Logger.warn "Flynn Auto-Scale: A scale event was stopped due to being above the maximum instance count"
+            return
+          end
+        elsif instances < 1 || instances > 2
+          Logger.warn "Flynn Auto-Scale: A scale event was stopped due to exceeding the default values"
+          return
+        end
+        # Okay, we checked, it's a good scaling event, hold on to your butts.
+        FlynnAutoScale::ScalingEvent.create(process_type: process, event_type: event_type, instances: instances)
+        `flynn -a #{ENV['FLYNN_SETUP_CLUSTER_APP_NAME']} scale #{process}=#{instances} -n`
+      end
+    end
+    # END Maintenance Section
+  end
+end

data/lib/flynn_auto_scale/engine.rb

@@ -0,0 +1,5 @@
+module FlynnAutoScale
+  class Engine < ::Rails::Engine
+    isolate_namespace FlynnAutoScale
+  end
+end

data/lib/flynn_auto_scale/version.rb

@@ -0,0 +1,3 @@
+module FlynnAutoScale
+  VERSION = '0.1.0'
+end

data/lib/tasks/flynn_auto_scale_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :flynn_auto_scale do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: flynn_auto_scale
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Lazaro Herrera
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: os
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A gem that allows your Rails apps to self-scale as they need more and
+  more resources under Flynn. Comes with an automated mode that can be used for 'day
+  to day' web hosting operations and a manual mode where you can control when scaling
+  operations take place (for scripts, background jobs and anything else)
+email:
+- lazherrera@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- app/assets/config/flynn_auto_scale_manifest.js
+- app/assets/javascripts/flynn_auto_scale/application.js
+- app/assets/stylesheets/flynn_auto_scale/application.css
+- app/controllers/flynn_auto_scale/application_controller.rb
+- app/helpers/flynn_auto_scale/application_helper.rb
+- app/jobs/flynn_auto_scale/application_job.rb
+- app/mailers/flynn_auto_scale/application_mailer.rb
+- app/models/flynn_auto_scale/application_record.rb
+- app/models/flynn_auto_scale/scaling_event.rb
+- app/views/layouts/flynn_auto_scale/application.html.erb
+- config/initializers/flynn_auto_scale.rb
+- config/routes.rb
+- db/migrate/20170530115155_create_flynn_auto_scale_scaling_events.rb
+- db/migrate/20170530121037_add_process_type_to_scaling_event.rb
+- lib/flynn_auto_scale.rb
+- lib/flynn_auto_scale/engine.rb
+- lib/flynn_auto_scale/version.rb
+- lib/tasks/flynn_auto_scale_tasks.rake
+homepage: https://github.com/WriteCodeEveryday/flynn_auto_scale
+licenses:
+- CC0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: A gem that allows your Rails apps to self-scale as they need more and more
+  resources under Flynn. Comes with an automated mode that can be used for 'day to
+  day' web hosting operations and a manual mode where you can control when scaling
+  operations take place (for scripts, background jobs and anything else)
+test_files: []