ridoku 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4552e8d282baa6c182c87dcec67db693da9e05da
+  data.tar.gz: 6cc461fe71080bdd283f3562f27fcf98911e6852
+SHA512:
+  metadata.gz: f5e25c2acec10b5de7f3e7b8676337a3f01774636e8d5427116e2b053a9a80b3f8acce7fbfad6aa378e23d22335b298ff73f9b4b0031fcd750decee7b04a9063
+  data.tar.gz: ca550fe9503c589089ec3c2315d50c15fdd971b0bb7efc47dae03d44cee21c420b02f7a1a4be7af73afa280c8b5bc44762fdc6614b0df2a1a97a99f7e811c6c8

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+
+*.sublime-*
+
+ridoku/.cookbook*
+.ridokurc

data/AUTHORS

@@ -0,0 +1,3 @@
+AUTHORS
+
+- Terry Meacham <zv1n.fire at gmail.com>

data/Gemfile

@@ -0,0 +1,24 @@
+source 'https://rubygems.org'
+ruby '2.1.2'
+#ruby-gemset=ridoku
+
+# OpsWorks control SDK + some IAM stuff.
+gem 'aws-sdk'
+
+# For fetching logs from S3.
+gem 'rest-client'
+gem 'rake'
+
+gem 'require_all'
+
+gem 'awesome_print'
+
+# JSON.load for the config file and for processing OpsWorks output.
+gem 'json'
+
+# Used for interacting with git
+gem 'rugged'
+
+# for deep merge
+gem 'activesupport'
+gem 'activesupport-inflector'

data/Gemfile.lock

@@ -0,0 +1,45 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    activesupport-inflector (0.1.0)
+    awesome_print (1.2.0)
+    aws-sdk (1.50.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    i18n (0.6.11)
+    json (1.8.1)
+    mime-types (2.3)
+    mini_portile (0.6.0)
+    minitest (5.4.0)
+    netrc (0.7.7)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    rake (10.3.2)
+    require_all (1.3.2)
+    rest-client (1.7.2)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rugged (0.21.0)
+    thread_safe (0.3.4)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  activesupport-inflector
+  awesome_print
+  aws-sdk
+  json
+  rake
+  require_all
+  rest-client
+  rugged

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2013, Terry Meacham
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,271 @@
+# *ridoku* v0.8
+
+*ridoku* is intended to be a set of scripts for replacing a Heroku work flow
+with AWS OpsWorks.  It requires some manual configuration at the moment in 
+AWS OpsWorks and the IAM Control Panel.
+
+## User Configuration
+
+You must add a user or add full permission to OpsWorks to your existing user.  
+The easiest way to do this is by following [this guide](http://docs.aws.amazon.com/opsworks/latest/userguide/opsworks-security-users.html).  Once you have this
+completed, you should be able to use `ridoku` to manage certain aspects of
+your app deployment.
+
+## Stack Configuration
+
+Currently, *ridoku* only works with the Rails Application Stack (rather, its
+only been tested on the stack, and several layer-actions specify 'rails-app'
+type, so your mileage may vary).
+
+Also, when developing the scripts, I was using a Rails app with a custom
+PostgreSQL, so the use of the OpsWorks MySQL layer has not been tested, but
+probably won't be affected (it will probably only limit the use of the
+[ridoku db](#db) command set for application management).
+
+Once the Stack has been created and instances added using the OpsWorks GUI, 
+you should be able to start using *ridoku* to make edits to your database and
+environment information, as well as running recipes and commands on the stack.
+
+Sadly, at this point, the OpsWorks GUI is still required (stack config, layers,
+etc are manual atm).
+
+## Custom Cookbooks
+
+Currently, the Ridoku custom cookbooks are also required to ensure that the
+environment is the same as is expected by a Heroku application.
+
+## Quickstart
+
+If you have a Stack that is already configured to use *ridoku*, this section
+gives you a quick run-down of commands necessary for Application management.
+
+(`rid` can also be used as an alias for `ridoku`)
+
+Each command below expects you to have run:
+
+```
+$ ridoku --set-stack YourStack
+$ ridoku --set-app YourApp
+$ ridoku --set-backup-bucket YourBackupBucket
+```
+
+The switches `--app app-name` and `--stack stack-name` can be used in any given
+commandline to override defaults.
+
+### Deploy/Rollback
+
+`$ ridoku deploy`
+
+Deploys the application to all instances.
+Note that **HEAD** is used for the repository branch associated with this app.
+(This is currently only configurable in the OpsWorks console)
+
+`$ ridoku deploy:rollback`
+
+Rollback the application on all instances.
+
+### Database Backup
+
+These commands only work if you are using *ridoku* to manage databases.
+
+`$ ridoku backup:capture`
+
+Captures the current applications database and stores it to S3.
+
+
+```
+$ ridoku backup:list
+$ ridoku backup:capture
+$ ridoku backup:restore <backup name>
+```
+
+Shows all existing database backups for the specified application,
+captures a backup (safety first!), then restores the specified database backup.
+
+### Environment
+
+*compare to `heroku config`*
+
+All changes to the environment require an application `deploy` to take effect.
+The Revision provider is used in the *ridoku* deployment cookbooks.  As a result,
+multiple deploy commands can be issued in a row without depleting the `rollback`
+capability (which is limited to 5 total rollbacks).
+
+`$ ridoku env`
+
+Displays the current applications runtime environment configuration.
+
+`$ ridoku env:set KEY:value KEY2:value2`
+
+Sets or updates the specified key/value pairs.
+
+`$ ridoku env:remove KEY`
+
+Removes the specified key/value pair.
+
+## *ridoku* commands
+
+Ridoku, 0.0.8
+
+`usage: ridoku [OPTIONS] command [command options]`
+
+### backup
+
+**TODO**
+
+### cook
+
+**TODO**
+
+### create
+
+**TODO**
+
+### db
+
+**TODO**
+
+### deploy
+
+**TODO**
+
+### domain
+
+**TODO**
+
+### dump
+
+**TODO**
+
+### env
+
+**TODO**
+
+### list
+
+**TODO**
+
+### packages
+
+**TODO**
+
+### run
+
+**TODO**
+
+### service
+
+**TODO**
+
+### workers
+
+**TODO**
+
+### Options:
+
+|CL Switch|Description|
+|---|---|
+|--debug/-D|Turn on debugging outputs (for AWS and Exceptions).|
+|--no-wait/-n|When issuing a command, do not wait for the command to return.|
+|--key/-k &lt;key&gt;|Use the specified key as the AWS_ACCESS_KEY|
+|--secret/-s &lt;secret&gt;|Use the specified secret as the AWS_SECRET_KEY|
+|--set-app/-A &lt;app&gt;|Use the specified App as the default Application.|
+|--set-backup-bucket/-B &lt;bucket name&gt;|Use the specified bucket name as the default Backup Bucket.|
+|--backup-bucket/-b &lt;bucket name&gt;|Use the specified bucket name as the current Backup Bucket.|
+|--set-stack/-S &lt;stack&gt;|Use the specified Stack as the default Stack.|
+|--set-user/-U &lt;user&gt;|Use the specified user as the default login user in 'run:shell'.|
+|--set-ssh-key/-K &lt;key file&gt;|Use the specified file as the default ssh key file.|
+|--ssh-key/-f &lt;key file&gt;|Override the default ssh key file for this call.|
+|--app/-a &lt;app&gt;|Override the default App name for this call.|
+|--stack/-t &lt;stack&gt;|Override the default Stack name for this call.|
+|--instances/-i &lt;instances&gt;|Run command on specified instances; valid delimiters: ',' or ':'|
+|--user/-u &lt;user&gt;|Override the default user name for this call.|
+|--comment/-m &lt;message&gt;|Optional for: deploy|
+|--domains/-d &lt;domains&gt;|Optional for: create:app. Add the specified domains to the newly created application.|
+|--layer/-l|**TODO**|
+|--repo/-r|**TODO**|
+|--service-arn/-V|**TODO**|
+|--instance-arn/-N|**TODO**|
+|--practice/-p|**TODO**|
+|--wizard/-w|**TODO**|
+
+## Configuration Wizard:
+
+In order to get ridoku configured with your OpsWorks account, Ridoku must
+collect pertinent required info. The wizard can be run at any time after the
+first with the command line option of `--wizard`.
+
+### Values to be configured:
+
+#### ssh_key:
+     
+Path to the SSH key to be used for git repositories
+(cook books, apps, etc).  It is recommended that this be generated
+separately from your personal SSH keys so that they can be revoked
+effecting other logins.
+
+#### service_role_arn:
+
+If a valid service_role_arn cannot be found, Ridoku will attempt to
+generate one for you.  If you've already used OpsWorks, Ridoku should be
+able to find the necessary Roles for you.
+
+#### instance_role_arn:
+
+If a valid instance_role_arn cannot be found, Ridoku will attempt to
+generate one for you.  If you've already used OpsWorks, Ridoku should be
+able to find the necessary Roles for you.
+
+
+## Apps and Stacks:
+
+Amazon OpsWorks similarly to Heroku, but, because you manage all the resources,
+you'll have to provide a bit more information than you do to Heroku in order
+for this commandline utility to assist.
+
+### Stacks:
+  The technology stack to use for a particular type of application.
+
+Heroku probably has a similar structure internally, because they allow
+you to use any number of program Stacks (Rails, PHP, Python, Go, etc).
+The difference is that now in OpsWorks you control the stack environment,
+where on Heroku you did not.
+
+If you have a stack configured, you can view pertinent information using
+
+`$ ridoku list:stacks`
+
+This will display the stacks that are currently associated with your AWS
+account.  To
+
+To set the specific stack to use:
+
+`$ ridoku --set-stack <stackname>`
+
+To set a single run stack or override the default:
+
+`$ ridoku --stack <stackname> --app <appname> command…`
+
+### Apps:
+  The actual application which runs on the technology stack.
+
+This is what you have control over on Heroku.  You can customize the app
+domains, database information, environment, etc, on a per-application
+basis.  The same goes for OpsWorks.
+
+To set the default app to use:
+
+`$ ridoku --set-app <stackname>`
+
+To set a specific run app or override the default:
+
+`$ ridoku --stack <stackname> --app <appname> command…`
+
+## Future
+
+I would like to get this to the point of a fully functional Heroku replacement.
+Adding a standard `Rails` stack using the standard `LB -> Web Server*N <-> DB` stack
+layout should be fairly easily accomplished.
+
+If you have any issues when attempting to use this toolchain, please feel free
+to submit a pull request.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/rid

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+load File.join(File.dirname(__FILE__), 'ridoku')

data/bin/ridoku

@@ -0,0 +1,350 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler'
+# Bundler.require(:default)
+
+# add self to libpath
+bin_file = Pathname.new(__FILE__).realpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+require 'getoptlong'
+require 'io-colorize'
+require 'helpers'
+
+require 'ridoku'
+require 'options'
+
+require 'require_all'
+require_rel '../lib/ridoku'
+
+RUNCOM = "#{ENV['HOME']}/.ridokurc"
+
+config = {
+  access_key_id: ENV['AWS_ACCESS_KEY'],
+  secret_access_key: ENV['AWS_SECRET_KEY']
+}
+
+@validations = []
+
+def validate(opt, arg, &block)
+  @validations << {
+    option: opt,
+    argument: arg,
+    block: block
+  }
+end
+
+def validate_arguments
+  @validations.each do |val|
+    val[:block].call(val[:option], val[:argument])
+  end
+end
+
+def print_options
+  $stderr.puts 'Options:'
+  Ridoku.options.each do |opt|
+    $stderr.puts "  #{opt[0]}/#{opt[1]} #{opt[3]}"
+    $stderr.puts
+  end
+end
+
+def print_commands
+  $stderr.puts <<-EOF
+Commands:
+  #{$stderr.colorize(Ridoku.commands.join("\n  "), :bold)}
+
+EOF
+end
+
+def print_help(full = false)
+  $stderr.puts <<-EOF
+Ridoku, #{Ridoku::VERSION}
+usage: ridoku [OPTIONS] command [command options]
+
+EOF
+  print_commands
+
+  if full
+    print_options
+
+    $stderr.puts <<-EOF
+  #{Ridoku::ConfigWizard.help_text}
+
+  Apps and Stacks:
+
+  Amazon OpsWorks similarly to Heroku, but, because you own all the resources,
+  you'll have to provide a bit more information than you do to Heroku in order
+  for this commandline utility to assist.
+
+    Stacks:  The technology stack to use for a particular type of application.
+
+      Heroku probably has a similar structure internally, because they allow
+      you to use any number of program Stacks (Rails, PHP, Python, Go, etc). 
+      The difference is that now in OpsWorks you control the stack environment,
+      where on Heroku you did not.
+
+      If you have a stack configured, you can view pertinent information using
+      
+      $ ridoku list:stacks
+
+      This will display the stacks that are currently associated with your AWS
+      account.  To 
+
+      To set the specific stack to use:
+
+      $ ridoku --set-stack <stackname>
+
+      To set a single run stack or override the default:
+
+      $ ridoku --stack <stackname> --app <appname> command...
+
+    Apps:  The actual application which runs on the technology stack.
+
+      This is what you have control over on Heroku.  You can customize the app
+      domains, database information, environment, etc, on a per-application
+      basis.  The same goes for OpsWorks.
+
+      To set the default app to use:
+
+      $ ridoku --set-app <stackname>
+
+      To set a specific run app or override the default:
+
+      $ ridoku --stack <stackname> --app <appname> command...
+  EOF
+  end
+end
+
+opts = GetoptLong.new(
+  *Ridoku.options.map { |opt| opt[0..2] }
+)
+
+opts.quiet = true
+
+Ridoku::Base.load_config(RUNCOM)
+
+begin
+  opts.each do |opt, arg|
+    case opt
+    when '--debug'
+      Ridoku::Base.config[:debug] = true
+
+    when '--practice'
+      Ridoku::Base.config[:practice] = true
+
+    when '--key'
+      ENV['AWS_ACCESS_KEY'] = arg
+
+    when '--secret'
+      ENV['AWS_SECRET_KEY'] = arg
+
+    when '--set-app'
+      Ridoku::Base.config[:app] = arg
+      Ridoku::Base.save_config(RUNCOM)
+
+      puts "Default app set to: #{arg}"
+      exit 0 unless ARGV.length > 0
+    when '--set-stack'
+      Ridoku::Base.config[:stack] = arg
+      Ridoku::Base.save_config(RUNCOM)
+
+      puts "Default stack set to: #{arg}"
+      exit 0 unless ARGV.length > 0
+    when '--set-user'
+      Ridoku::Base.config[:shell_user] = arg
+      Ridoku::Base.save_config(RUNCOM)
+
+      puts "Default shell user set to: #{arg}"
+      exit 0 unless ARGV.length > 0
+    when '--set-backup-bucket'
+      Ridoku::Base.config[:backup_bucket] = arg
+      Ridoku::Base.save_config(RUNCOM)
+
+      puts "Default S3 backup bucket set to: #{arg}"
+      exit 0 unless ARGV.length > 0
+    when '--set-ssh-key'
+      Ridoku::Base.config[:ssh_key] = arg
+
+      validate(opt,arg) do |option, argument|
+        if File.exists?(argument)
+          Ridoku::Base.save_config(RUNCOM)
+        else
+          $stderr.puts 'Specified SSH Key file does not exist!'
+          print_help
+          exit 1
+        end
+      end
+
+      puts "Default shell user set to: #{arg}"
+    when '--ssh-key'
+      Ridoku::Base.config[:ssh_key] = arg
+
+    when '--repo'
+      Ridoku::Base.config[:repo] = arg
+
+    when '--backup-bucket'
+      Ridoku::Base.config[:backup_bucket] = arg
+
+    when '--domains'
+      Ridoku::Base.config[:domains] = arg.split(%r(\||;|:|,))
+
+    when '--service-arn'
+      Ridoku::Base.config[:service_arn] = arg
+
+    when '--rails-env'
+      Ridoku::Base.config[:rails_env] = arg
+
+    when '--instance-arn'
+      Ridoku::Base.config[:instance_arn] = arg
+
+    when '--wizard'
+      Ridoku::Base.config[:wizard] = true
+
+    when '--lines'
+      unless arg.match(/^[0-9]+$/)
+        $stderr.puts '--lines must be a number!'
+        exit 1
+      end
+      Ridoku::Base.config[:wizard] = arg
+
+    when '--stack'
+      Ridoku::Base.config[:stack] = arg
+
+    when '--force'
+      Ridoku::Base.config[:force] = true
+
+    when '--migrate'
+      Ridoku::Base.config[:migrate] = true
+
+    when '--user'
+      Ridoku::Base.config[:shell_user] = arg
+
+    when '--no-wait'
+      Ridoku::Base.config[:wait] = false
+
+    when '--layer'
+      Ridoku::Base.config[:layer] = arg
+
+    when '--app'
+      Ridoku::Base.config[:app] = arg
+
+    when '--comment'
+      Ridoku::Base.config[:comment] = arg
+
+    when '--instances'
+      Ridoku::Base.config[:instances] = arg.split(%r(,|\||;|:))
+
+      validate(opt, Ridoku::Base.config[:instances]) do |option, argument|
+        unless Ridoku::Base.valid_instances?(argument)
+          $stderr.puts 'Invalid instances provided.'
+
+          validinst = []
+          Ridoku::Base.instances.each do |inst|
+            next unless inst[:status] != 'stopped'
+            val = "#{inst[:hostname]} [#{inst[:status]}]"
+            validinst << $stderr.colorize(val, [:bold, :green])
+          end
+
+          invalidinst = []
+          Ridoku::Base.instances.each do |inst|
+            next unless inst[:status] != 'online'
+            val = "#{inst[:hostname]} [#{inst[:status]}]"
+            invalidinst << $stderr.colorize(val, [:bold, :red])
+          end
+
+          $stderr.puts 'Valid Instances:'
+          $stderr.puts validinst
+          $stderr.puts 'Invalid Instances:'
+          $stderr.puts invalidinst
+
+          puts 'Run the help command to see more.'
+          exit 1
+        end
+      end
+    else
+      print_help
+      exit 1
+    end
+  end
+rescue => e
+  puts e.to_s
+  print_help
+  exit 1
+end
+
+if Ridoku::Base.config[:debug]
+  # log to standard out, strip all of the Ruby logger standard prefixes
+  logger = Logger.new($stdout)
+  logger.formatter = proc { |severity, datetime, progname, msg| msg }
+
+  config.merge!({
+    logger: logger,
+    log_formatter: AWS::Core::LogFormatter.colored,
+    http_wire_trace: true
+  })
+end
+
+begin 
+  AWS.config(config)
+
+  unless ENV.key?('AWS_ACCESS_KEY') && ENV.key?('AWS_SECRET_KEY')
+    puts 'AWS_ACCESS_KEY and AWS_SECRET_KEY must be specified in your environment.'
+    exit 1
+  end
+
+  if !Ridoku::Base.config[:local_init] || Ridoku::Base.config[:wizard]
+    begin
+      wizard = Ridoku::ConfigWizard.new
+      wizard.run
+    rescue Ridoku::InvalidConfig => e
+      $stderr.puts "#{e.error.to_s.capitalize} #{e.type.to_s} specified."
+      $stderr.puts 'Use the `list` command to see relavent info.'
+      exit 1
+    end
+  end
+
+  validate_arguments
+
+  unless ARGV.length > 0
+    puts 'No arguments specified.'
+    print_help
+    exit 1
+  end
+
+  Ridoku::Base.config[:command] = ARGV.shift.split(':')
+
+  if Ridoku::Base.config[:command][0] == 'help'
+    print_help(true)
+    exit 0
+  end
+
+  begin
+    command = Ridoku.const_get(
+      Ridoku::Base.config[:command].first.capitalize
+    ).new
+  rescue => e
+    $stderr.puts "Invalid command specified: #{Ridoku::Base.config[:command][0]}"
+    puts e.to_s if Ridoku::Base.config[:debug]
+    print_help
+    exit 1
+  end
+
+  begin
+    command.run
+  rescue Ridoku::InvalidConfig => e
+    $stderr.puts "#{e.error.to_s.capitalize} #{e.type.to_s} specified."
+    $stderr.puts 'Use the `list` command to see relavent info.'
+    exit 1
+  rescue Ridoku::NoSshAccess
+    $stderr.puts 'Your user does not have access to ssh on the specified stack.'
+    exit 1
+  rescue ArgumentError => e
+    raise e if Ridoku::Base.config[:debug]
+    $stderr.puts e.to_s
+  end
+rescue AWS::OpsWorks::Errors::UnrecognizedClientException => e
+  $stderr.puts e.to_s
+  $stderr.puts 'Check to ensure that AWS_ACCESS_KEY and AWS_SECRET_KEY are set properly.'
+  exit 1
+end