opzworks 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a7a099fc80bd069a428e204ed8b8ae80e970405
+  data.tar.gz: 902044b0b2bde27c5a8bbd45fcf242446c3be66a
+SHA512:
+  metadata.gz: d9fae78a2acc179277714c933268b76adf7f4e671e37a0971a3af61745972887b0806911021aac80e77efd8856205cb99afe6e0dfd229560f21bc8ae652f38d3
+  data.tar.gz: f7014b5ca89eeb1e2b881bc7aee17f6020ba37c8679681bc9865a94f894f0dea1d1a8f82a9f2753cff7261d6b01e1e2f22671b01f3da928f4eae8c10d6ba479b

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/CHANGELOG.md

@@ -0,0 +1,30 @@
+changelog
+=========
+
+0.3.0
+-----
+* attempt to clone opsworks-${project} repo if not found
+
+0.2.4
+-----
+* provide ability to change the amount of diff context via --context {int} switch to json command
+
+0.2.3
+-----
+* provide --private option for ssh to allow use of private ips (defaults to public)
+
+0.2.2
+-----
+* big speed improvement for ssh by removing unecessary aws calls
+
+0.2.1
+-----
+* documentation enhancements
+
+0.2.0
+-----
+* elastic command support
+
+0.1.0
+-----
+* initial release with berks/json/ssh support

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in opzworks.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Adam Lindberg
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,212 @@
+# OpzWorks CLI
+
+Command line interface for managing AWS OpsWorks chef cookbooks and stack json, as well
+as other OpsWorks centric tasks such as generating ssh configs for OpsWorks instances.
+
+## Build Status
+
+[![Circle CI](https://circleci.com/gh/mapzen/opzworks.svg?style=svg)](https://circleci.com/gh/mapzen/opzworks)
+
+## Third party requirements:
+
+Aside from a recent version of ruby:
+
+* git
+* [ChefDK](https://downloads.chef.io/chef-dk/)
+
+## Installation
+
+Install for use on the command line (requires ruby and rubygems): `gem install opzworks`
+
+Then run `opzworks --help`
+
+To use the gem in a project, add `gem 'opzworks'` to your Gemfile, and then execute: `bundle`
+
+To build locally from this repository: `rake install`
+
+## Commands
+
+Run `opzworks` with one of the following commands:
+
+#### ssh
+
+Generate and update SSH configuration files.
+
+Host names are based off the stack naming convention, `project_name::env::region`. The default
+is to use public instance IPs (or elastic ip if one is assigned). Passing the `--private` option
+will instead use instance private IPs.
+
+For example, if we have a host 'api1' in the stack apiaxle::prod::us-east, the
+resultant hostname will be `api1-apiaxle-prod-us-east`
+
+By default, `opsworks ssh` will iterate over all stacks. If you wish to restrict the stacks
+it searches, simply pass the stack name (or a partial match) as an argument:
+
+`opzworks ssh myproject::prod`
+
+If you wanted to automatically scrape all your stacks to populate your ssh config, and
+you don't want to use the `--update` flag (which will overwrite the entire file contents),
+you could do something like:
+
+* add a crontab entry similar to: `0 * * * * /bin/bash -l -c /path/to/opzworks-ssh.sh`
+* create `/path/to/opzworks-ssh.sh`:
+
+```bash
+# this script reads .ssh/config, drops anything after the matched line,
+#   then generates a list of opsworks hosts and appends them to the file.
+gsed -i '/OPSWORKS_CRON_LINE_MATCH/q' ~/.ssh/config
+opzworks ssh >>~/.ssh/config
+```
+
+Note this example assumes the use of a gnu sed-like utility, which on OSX means
+installing gnu sed (`brew install gsed` if you're using homebrew). On Linux, simply
+change `gsed` to `sed`.
+
+Add the following line to the bottom of your existing ~/.ssh/config:
+
+`# OPSWORKS_CRON_LINE_MATCH`
+
+#### elastic
+
+Perform [start|stop|bounce|rolling] operations on an Elastic cluster.
+
+The host from which this command is originated will need to have access to the the target
+systems via private IP and assumes port 9200 is open and available.
+
+This is a very rough implementation!
+
+#### json
+
+Update stack custom JSON.
+
+#### berks
+
+Build the berkshelf for a stack, upload the tarball to S3, trigger `update_custom_cookbooks` on the stack.
+
+## Configuration
+
+The gem reads information from `~/.aws/config`, or from the file referenced by
+the `AWS_CONFIG_FILE` environment variable. It should already look something like this:
+
+    [default]
+    aws_access_key_id     = ilmiochiaveID
+    aws_secret_access_key = ilmiochiavesegreto
+    region                = us-east-1
+    output                = json
+
+If you want the gem to read from an environment other than 'default', you can do so
+by exporting the `AWS_PROFILE` environment variable. It should be set to whatever profile
+name you have defined that you want to use in the config file.
+
+Add the following section to `~/.aws/config`:
+
+    [opzworks]
+    ssh-user-name         = <MY SSH USER NAME>
+    berks-repository-path = <PATH TO OPSWORKS BERKSHELF REPOSITORIES>
+    berks-github-org      = <GITHUB ORG THAT YOUR OPSWORKS REPOSITORIES EXIST UNDER>
+    berks-s3-bucket       = <AN EXISTING S3 BUCKET>
+
+The `ssh-user-name` value should be set to the username you want to use when
+logging in remotely, most probably the user name from your _My Settings_ page
+in OpsWorks.
+
+The `berks-repository-path` should point to a base directory in which your opsworks
+git repositories for each stack will live.
+
+The `berks-s3-bucket` will default to 'opzworks' if not set. You need to create the
+the bucket manually (e.g. `aws s3 mb s3://opsworks-cookbook-bucket`).
+
+The `berks-github-org` setting is used if you try to run `berks` or `json` on a stack, and
+the local opsworks-${project} repo isn't found. In this event, the code will attempt to clone
+the repo into `berks-repository-path` and continue.
+
+Additional option are:
+
+`berks-base-path`, which is the temporary base directory where the berkshelf will be
+built. Defaults to /tmp.
+
+`berks-tarball-name`, which is the name of the tarball that will be uploaded to S3. Defaults to cookbooks.tgz.
+
+## Setup Conventions/Workflow for Berks/JSON Commands
+
+![workflow](img/flow.png)
+
+This gem makes a number of assumptions in order to enforce a specific workflow. First among them is
+the Opsworks stack naming convection. This will need to adhere toe the following format:
+
+    PROJECT::ENV::REGION
+
+If PROJECT will be comprised of multiple words, they should be joined with underscores, e.g.
+
+    my_awesome_rails_app::prod::us-east
+
+So for example, if you have an Elastic cluster in dev and prod in us-east, and dev in us-west:
+
+    elastic::dev::us-east
+    elastic::dev::us-west
+    elastic::prod::us-east
+
+The next workflow that must be conformed to is berkshelf management. In this context, that means a git
+repository that conforms to the following setup:
+
+    opsworks-project
+
+Inside that repository, you will have branches that match each of your environments and regions.
+
+So in our Elastic example, you would have the following setup:
+
+* a git repository called opsworks-elastic
+* branches in that repository called dev-us-east, dev-us-west and prod-us-east
+
+In each of those branches, you should have the following:
+
+* Berksfile
+* stack.json (if you want to maintain the stack json using the `opzworks json` utility)
+
+The Berksfile will look similar to the following. If you're familiar with Berkshelf, there's nothing
+new here:
+
+```ruby
+source 'https://api.berkshelf.com'
+
+# opsworks
+cookbook 'apache2' , github: 'aws/opsworks-cookbooks' , branch: 'release-chef-11.10' , rel: 'apache2'
+
+# external
+#
+cookbook 'lvm',           '= 1.0.8'
+cookbook 'sensu',         '= 2.10.0'
+cookbook 'runit',         '= 1.5.10'
+cookbook 'java',          '= 1.29.0'
+cookbook 'nodejs',        '= 2.1.0'
+cookbook 'elasticsearch', '= 0.3.13'
+cookbook 'chef_handler',  '= 1.1.6'
+
+# mapzen wrappers
+#
+cookbook 'mapzen_sensu_clients',  git: 'git@github.com:mapzen/chef-mapzen_sensu_clients', tag: '0.12.0'
+cookbook 'mapzen_elasticsearch',  git: 'git@github.com:mapzen/chef-mapzen_elasticsearch', tag: '0.16.3'
+cookbook 'mapzen_logstash',       git: 'git@github.com:mapzen/chef-mapzen_logstash',      tag: '0.13.1'
+cookbook 'mapzen_graphite',       git: 'git@github.com:mapzen/chef-mapzen_graphite',      tag: '0.6.0'
+cookbook 'mapzen_pelias',         git: 'git@github.com:mapzen/chef-mapzen_pelias',        tag: '0.34.2'
+```
+
+If we placed that Berkshelf file in opsworks-elastic, in the prod-us-east branch, we would run `opzworks berks elastic::prod::us-east`, which would do the following:
+
+* build the berkshelf locally
+* push the resultant cookbook tarball to: s3://opzworks/elastic-prod-us-east/cookbooks.tgz
+* run `update_custom_cookbook` on the stack (unless you pass the `--no-update` flag)
+
+Your stack should be configured to use a berkshelf from an S3 archive. The url will look as below:
+
+    https://s3.amazonaws.com/opzworks/elastic-prod-us-east/cookbooks.tgz
+
+You'll need to set up an IAM user or users with permission to access the location.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+
+namespace :test do
+  desc 'Run tests'
+  task :syntax do
+    puts 'Running rubocop'
+    sh 'rubocop .'
+  end
+end
+
+task default: 'test:syntax'

data/bin/opzworks

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+bin_file = Pathname.new(__FILE__).realpath
+$LOAD_PATH.unshift File.expand_path('../../lib', bin_file)
+
+require 'opzworks/cli'
+
+OpzWorks::CLI.start

data/circle.yml

@@ -0,0 +1,6 @@
+machine:
+  ruby:
+    version: 2.3.0
+test:
+  override:
+    - bundle exec rake

data/lib/opzworks/cli.rb

@@ -0,0 +1,47 @@
+require 'trollop'
+require 'opzworks'
+
+module OpzWorks
+  class CLI
+    def self.start
+      commands = %w(ssh json berks elastic)
+
+      Trollop.options do
+        version "opzworks #{OpzWorks::VERSION} (c) #{OpzWorks::AUTHORS.join(', ')}"
+        banner <<-EOS.unindent
+          usage: opzworks [COMMAND] [OPTIONS...]
+
+          #{OpzWorks::SUMMARY}
+
+          Commands
+            ssh  #{OpzWorks::Commands::SSH.banner}
+            json #{OpzWorks::Commands::JSON.banner}
+            berks #{OpzWorks::Commands::BERKS.banner}
+            elastic #{OpzWorks::Commands::ELASTIC.banner}
+
+          For help with specific commands, run:
+            opzworks COMMAND -h/--help
+
+          Options:
+        EOS
+        stop_on commands
+      end
+
+      command = ARGV.shift
+      case command
+      when 'ssh'
+        OpzWorks::Commands::SSH.run
+      when 'json'
+        OpzWorks::Commands::JSON.run
+      when 'berks'
+        OpzWorks::Commands::BERKS.run
+      when 'elastic'
+        OpzWorks::Commands::ELASTIC.run
+      when nil
+        Trollop.die 'no command specified'
+      else
+        Trollop.die "unknown command: #{command}"
+      end
+    end
+  end
+end

data/lib/opzworks/commands/berks.rb

@@ -0,0 +1,147 @@
+require 'aws-sdk'
+require 'trollop'
+require 'opzworks'
+require 'rainbow/ext/string'
+
+require_relative 'include/run_local'
+require_relative 'include/populate_stack'
+require_relative 'include/manage_berks_repos'
+
+module OpzWorks
+  class Commands
+    class BERKS
+      def self.banner
+        'Build the stack berkshelf'
+      end
+
+      def self.run
+        options = Trollop.options do
+          banner <<-EOS.unindent
+            #{BERKS.banner}
+
+              opzworks berks stack1 stack2 ...
+
+            The stack name can be passed as any unique regex. If there is
+            more than one match, it will simply be skipped.
+
+            Options:
+          EOS
+          opt :update, 'Trigger update_custom_cookbooks on stack after uploading a new cookbook tarball.', default: true
+        end
+        ARGV.empty? ? Trollop.die('no stacks specified') : false
+
+        config = OpzWorks.config
+
+        aws_credentials_provider = Aws::SharedCredentials.new(profile_name: config.aws_profile)
+        s3 = Aws::S3::Resource.new(region: config.aws_region, credentials: aws_credentials_provider)
+
+        opsworks = Aws::OpsWorks::Client.new(region: config.aws_region, profile: config.aws_profile)
+        response = opsworks.describe_stacks
+
+        # loops over inputs
+        ARGV.each do |opt|
+          populate_stack(opt, response)
+          next if @populate_stack_failure == true
+
+          manage_berks_repos
+          next if @berks_repo_failure == true
+
+          berks_cook_path  = config.berks_base_path || '/tmp'
+          cook_path        = "#{berks_cook_path}/#{@project}-#{@branch}"
+          install_path     = "#{cook_path}" + '/' + "cookbooks-#{@project}-#{@branch}"
+          cookbook_tarball = config.berks_tarball_name || 'cookbooks.tgz'
+          cookbook_upload  = "#{cook_path}" + '/' "#{cookbook_tarball}"
+          s3_bucket        = config.berks_s3_bucket || 'opzworks'
+          opsworks_berks   = 'Berksfile.opsworks'
+          overrides        = 'overrides'
+
+          # berks
+          #
+          puts 'Running berks install'.foreground(:blue)
+          run_local <<-BASH
+            cd #{@target_path}
+            berks update
+          BASH
+          run_local <<-BASH
+            cd #{@target_path}
+            berks vendor #{install_path}
+          BASH
+
+          # if there's a Berksfile.opsworks, push it up to let nodes build their cookbook
+          #   repository from its contents
+          #
+          if File.file?("#{@target_path}/#{opsworks_berks}")
+            puts 'Copying opsworks Berksfile into place'.foreground(:blue)
+            FileUtils.copy("#{@target_path}/#{opsworks_berks}", "#{install_path}/Berksfile")
+          end
+
+          # if there's an overrides file, just pull it and stuff the contents into the
+          #   upload repo; the line is assumed to be a git repo. This is done to override
+          #   opsworks templates without destroying the upstream cookbook.
+          #
+          #   For example, to override the default nginx cookbook's nginx.conf, create a git
+          #     repo with the directory structure nginx/templates/default and place your
+          #     custom nginx.conf.erb in it.
+          #
+          if File.file?("#{@target_path}/#{overrides}")
+            unless File.directory?("#{install_path}")
+              FileUtils.mkdir_p("#{install_path}")
+            end
+            File.open("#{@target_path}/#{overrides}") do |f|
+              f.each_line do |line|
+                puts "Copying override #{line}".foreground(:blue)
+                `cd #{install_path} && git clone #{line}`
+              end
+            end
+          end
+
+          puts 'Committing changes and pushing'.foreground(:blue)
+          system "cd #{@target_path} && git commit -am 'berks update'; git push origin #{@branch}"
+
+          puts 'Creating tarball of cookbooks'.foreground(:blue)
+          FileUtils.mkdir_p("#{cook_path}")
+          run_local "tar czf #{cookbook_upload} -C #{install_path} ."
+
+          # upload
+          #
+          puts 'Uploading to S3'.foreground(:blue)
+
+          begin
+            obj = s3.bucket(s3_bucket).object("#{@s3_path}/#{cookbook_tarball}")
+            obj.upload_file("#{cookbook_upload}")
+          rescue StandardError => e
+            puts "Caught exception while uploading to S3 bucket #{s3_bucket}: #{e}".foreground(:red)
+            puts 'Cleaning up before exiting'.foreground(:blue)
+            FileUtils.rm("#{cookbook_upload}")
+            FileUtils.rm_rf("#{install_path}")
+            abort
+          else
+            puts "Completed successful upload of #{@s3_path}/#{cookbook_tarball} to #{s3_bucket}!".foreground(:green)
+          end
+
+          # cleanup
+          #
+          puts 'Cleaning up'.foreground(:blue)
+          FileUtils.rm("#{cookbook_upload}")
+          FileUtils.rm_rf("#{install_path}")
+          puts 'Done!'.foreground(:green)
+
+          # update remote cookbooks
+          #
+          if options[:update] == true
+            puts "Triggering update_custom_cookbooks for remote stack (#{@stack_id})".foreground(:blue)
+
+            hash = {}
+            hash[:comment]  = 'shake and bake'
+            hash[:stack_id] = @stack_id
+            hash[:command]  = { name: 'update_custom_cookbooks' }
+
+            opsworks.create_deployment(hash)
+          else
+            puts 'Update custom cookbooks skipped via --no-update switch.'.foreground(:blue)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opzworks/commands/elastic.rb

@@ -0,0 +1,103 @@
+require 'aws-sdk'
+require 'trollop'
+require 'faraday'
+require 'opzworks'
+require 'net/ssh'
+require 'net/ssh/multi'
+require 'rainbow/ext/string'
+
+require_relative 'include/elastic'
+
+module OpzWorks
+  class Commands
+    class ELASTIC
+      def self.banner
+        'Perform operations on an Elastic cluster'
+      end
+
+      def self.run
+        options = Trollop.options do
+          banner <<-EOS.unindent
+            #{ELASTIC.banner}
+
+              opzworks elastic stack1 stack2 ... [--start|--stop|--bounce|--rolling]
+
+            The stack name can be passed as any unique regex. If there is
+            more than one match, it will simply be skipped.
+
+            Options:
+          EOS
+          opt :start, 'Start Elastic', default: false
+          opt :stop, 'Stop Elastic', default: false
+          opt :bounce, 'Bounce (stop/start) Elastic', default: false
+          opt :rolling, 'Perform a rolling restart of Elastic', default: false
+        end
+        ARGV.empty? ? Trollop.die('no stacks specified') : false
+
+        optarr = []
+        options.each do |opt, val|
+          val == true ? optarr << opt : false
+        end
+        optarr.empty? ? Trollop.die('no options specified') : false
+
+        config = OpzWorks.config
+        @client = Aws::OpsWorks::Client.new(region: config.aws_region, profile: config.aws_profile)
+        response = @client.describe_stacks
+
+        # loops over inputs
+        ARGV.each do |opt|
+          es_get_input(opt, response)
+          next if @get_data_failure == true
+
+          case options[:rolling]
+          when true
+            # cycle through all the hosts, waiting for status
+            @ip_addrs.each do |ip|
+              puts "\n________________________________________________"
+              puts "Now operating on host #{ip}".foreground(:yellow)
+
+              es_enable_allocation(ip, 'none') if @disable_shard_allocation == true
+              sleep 2 if @disable_shard_allocation == true
+
+              es_service('restart', [ip])
+              es_wait_for_status(ip, 'yellow')
+              es_enable_allocation(ip, 'all') if @disable_shard_allocation == true
+              es_wait_for_status(ip, 'green')
+            end
+          end
+
+          case options[:start]
+          when true
+            es_service('start', @ip_addrs)
+
+            @ip_addrs.each do |ip|
+              es_wait_for_status(ip, 'green')
+            end
+          end
+
+          case options[:stop]
+          when true
+            # use the first host to disable shard allocation
+            es_enable_allocation(@ip_addrs.first, 'none') if @disable_shard_allocation == true
+            sleep 2 if @disable_shard_allocation == true
+
+            es_service('stop', @ip_addrs)
+          end
+
+          case options[:bounce]
+          when true
+            # use the first host to disable shard allocation
+            es_enable_allocation(@ip_addrs.first, 'none') if @disable_shard_allocation == true
+            sleep 2 if @disable_shard_allocation == true
+
+            es_service('restart', @ip_addrs)
+
+            es_wait_for_status(@ip_addrs.first, 'yellow')
+            es_enable_allocation(@ip_addrs.first, 'all') if @disable_shard_allocation == true
+            es_wait_for_status(@ip_addrs.first, 'green')
+          end
+        end
+      end
+    end
+  end
+end