qops 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a359bd27c24df1c4c124864fdc9d409bf43e453
+  data.tar.gz: 3d42a3143c38fde591eae2969d8abe4d0be36b20
+SHA512:
+  metadata.gz: 4ca05c0d395bbcadfc399ad7a52649f19ebf68520c09acd37bb56c0d86bb9c77cfe0c65dd7639a3c1e6b43c2f9b6f460158a9d94374129806f6832b03b1df78d
+  data.tar.gz: cfb7726f54f320898ba5c955babb1d1a17bdbcc5695612a740dbb347f743bd49a82fe51264b59184bd638d89484b73423ea7e101372752a821b3deed301eb93c

data/README.md

@@ -0,0 +1,103 @@
+# Qops : Quandl Operations Helper
+
+## Configuring Qops in your project
+
+1. In your repo's config directory create a file called opsworks.yml. See sample.
+2. Install gem `qops` to your system. Be sure to use gem install `qops`. Do not include it as part of the bundle of your repo as it is mean't to be run outside of the scope of your project.
+3. Run `qops list` to get a list of commands you can run.
+4. Run `qops help <command>` for more information on command argument options
+
+## FAQ:
+
+### Q: The `qops` gem is currently not public. How do I access it?
+
+Please add your personal gemfury source to the gem path to install it.
+
+### Q: For the `qops qops:instance:run_command` command, it provides two options: one is run commands against all instances of the stack all in once, one is run commands on each instances of the stack one by one randomly. How do I use this?
+
+When running commands one by one, between each execution of the command, there will be a delay. The delay is config by wait_deploy. By default it is 180 seconds when it is not defined. For now, run_command command will only support commends `setup` `configure` `install_dependencies` `update_dependencies`, since commands `update_custom_cookbooks` `deploy` was implemented before.
+
+### Q: How do I use QOPS to override env variables on my opsworks CHEF 11 stack?
+
+You can use the custom json flag for this. Example:
+
+```qops qops:instance:up -e staging -j '{ "deploy" : { "wikiposit" : { "environment_variables" : { "ENV_VARIABLE_TO_OVERIDE": "X" } } } }'```
+
+In this case the `wikiposit` is the opsworks app wikiposit. You will need to change this to whichever app you are deploying.
+
+See also: http://docs.aws.amazon.com/opsworks/latest/userguide/apps-environment-vars.html
+
+## Sample Config (with all options)
+
+```
+_daily_schedule: &daily_schedule
+  '13': 'on'
+  '14': 'on'
+  '15': 'on'
+  '16': 'on'
+  '17': 'on'
+  '18': 'on'
+  '19': 'on'
+  '20': 'on'
+  '21': 'on'
+  '22': 'on'
+
+_weekly_schedule: &weekly_schedule
+  monday: *daily_schedule
+  tuesday: *daily_schedule
+  wednesday: *daily_schedule
+  thursday: *daily_schedule
+  friday: *daily_schedule
+
+_default: &default
+  wait_iterations: 600 # Optional
+  command_log_lines: 100 # Optional
+  autoscale_type: ~ # Optional
+  region: us-east-1
+  app_name: 'wikiposit'
+  instance_type: 't2.small'
+  max_instance_duration: 86400 # Optional
+  clean_commands_to_ignore: ['configure', 'shutdown] # Optional: A list of opsworks commands to ignore when calculating that last run time for the clean command. Ignores `configure` and `shutdown` commands by default.  
+  cookbook_dir: cookbooks
+  cookbook_name: wikiposit
+  cookbook_version: "<%= IO.read(File.join(Dir.pwd, 'cookbooks/VERSION')).strip %>"
+  cookbook_s3_bucket: quandl-cookbooks
+
+staging:
+  <<: *default
+  deploy_type: :staging
+  stack_id: 1aec9354-e1bc-4f31-8627-2208f2382dcb
+  layer_id: 622555c4-fc07-4ff1-ba79-0a63fbd233f5
+  application_id: 15904509-ace3-4d78-923e-ea10b3b2d433 # Optional. Deploy command will not run without application ID.
+  subnet: subnet-0cde5d27
+  cookbook_s3_path: staging/my-app
+```
+
+## Sample Slack.yml
+
+If you create a `config/quandl/slack.yml` file as so slack messages can be enabled for deployments under various environments.
+
+```
+defaults: &defaults
+  webhook_url: https://hooks.slack.com/services/......
+  notifiers: &default_notifiers
+    cookbook:
+      channel: '#releases'
+      username: My App Cookbooks
+      icon_emoji: ':book:'
+    release:
+      channel: '#releases'
+      username: My App
+      icon_emoji: ':rocket:'
+    instance_up:
+      channel: '#releases'
+      username: My App
+      icon_emoji: ':chart_with_upwards_trend:'
+    instance_down:
+      channel: '#releases'
+      username: My App
+      icon_emoji: ':chart_with_downwards_trend:'
+
+development:
+  <<: *defaults
+```

data/bin/qops

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$LOAD_PATH.unshift(File.dirname(File.realpath(__FILE__)) + '/../lib')
+
+require 'thor/runner'
+require 'qops'
+
+$thor_runner = true
+Thor::Runner.start

data/lib/qops.rb

@@ -0,0 +1,32 @@
+require 'thor'
+require 'thor/group'
+require 'aws-sdk'
+require 'json'
+require 'fileutils'
+require 'active_support/all'
+require 'pp'
+require 'optparse'
+require 'erb'
+require 'rainbow'
+
+require 'quandl/slack'
+
+require_relative 'qops/environment'
+require_relative 'qops/helpers'
+require_relative 'qops/deployment/helpers'
+require_relative 'qops/deployment/app'
+require_relative 'qops/deployment/instances'
+require_relative 'qops/cookbook/cookbook'
+
+# Migrate this into quandl config project
+module Quandl
+  class Config < ::OpenStruct
+    cattr_accessor :environment
+
+    private
+
+    def project_environment
+      @_environment ||= @@environment || (defined?(Rails) ? ::Rails.env : nil) || ENV['RAILS_ENV'] || ENV['RAKE_ENV'] || ENV['QUANDL_ENV'] || 'default'
+    end
+  end
+end

data/lib/qops/cookbook/cookbook.rb

@@ -0,0 +1,160 @@
+class Qops::Cookbook < Thor
+  include Qops::Helpers
+
+  desc 'vendor', 'Generate vendor directory to contain the cookbooks'
+  def vendor
+    initialize_run
+    cleanup
+    Dir.chdir(config.cookbook_dir) do
+      system('berks vendor vendor -e opsworks')
+    end
+  end
+
+  desc 'package', 'Package the cookbooks into a zip file in vendor'
+  def package
+    initialize_run
+    Dir.chdir(config.cookbook_dir) do
+      remove_zip_files
+      system("zip -r #{artifact_name} vendor/*")
+    end
+  end
+
+  desc 'upload', 'Uploads cookbooks to s3'
+  def upload
+    initialize_run
+    s3.put_object(
+      bucket: config.cookbook_s3_bucket,
+      acl: 'private',
+      key: remote_artifact_file,
+      body: IO.read(local_artifact_file)
+    )
+  end
+
+  desc 'update_custom_json', 'Upload custom json to stack'
+  def update_custom_json
+    initialize_run
+    raw_json = File.read(File.join(config.cookbook_dir, config.cookbook_json))
+    json = JSON.parse(raw_json)
+
+    say(JSON.pretty_generate(json), :yellow)
+    if yes?("Are you sure you want to update the custom JSON for opsworks stack #{config.stack_id}?", :yellow)
+      config.opsworks.update_stack(
+        stack_id: config.stack_id,
+        custom_json: JSON.pretty_generate(json)
+      )
+      say('Updated!', :green)
+    else
+      say('You said no, so we\'re done here.', :yellow)
+    end
+  rescue JSON::ParserError
+    say('Check your JSON for errors!', :red)
+  end
+
+  desc 'update_stack_cookbooks', 'Runs the opsworks command to update custom cookbooks.'
+  def update_stack_cookbooks
+    initialize_run
+    if yes?("Are you sure you want to run the 'Update Custom Cookbooks' command on stack #{config.stack_id}?", :yellow)
+      run_opsworks_command(
+        stack_id: config.stack_id,
+        command: {
+          name: 'update_custom_cookbooks'
+        }
+      )
+      say('Updated!', :green)
+    else
+      say('You said no, so we\'re done here.', :yellow)
+      exit(-1)
+    end
+  end
+
+  desc 'release', 'Zip, package and update a new cookbook as a release.'
+  def release
+    vendor && package && upload && update_custom_cookbooks && update_stack_cookbooks
+
+    ping_slack('Quandl::Slack::Cookbook', 'Cookbook updated', 'success',
+               command: 'opsworks cookbook release',
+               status: 'success',
+               name: config.cookbook_name,
+               version: config.cookbook_version,
+               stack: config.stack_id
+              )
+
+    say('Released!', :green)
+  end
+
+  desc 'update_custom_cookbooks', 'Update the stack with the custom cookbooks!'
+  def update_custom_cookbooks
+    if yes?("Are you sure you want to update the custom the custom cookbook for opsworks stack #{config.stack_id}?", :yellow)
+      config.opsworks.update_stack(
+        stack_id: config.stack_id,
+        use_custom_cookbooks: true,
+        custom_cookbooks_source: {
+          type: 's3',
+          url: "https://s3.amazonaws.com/#{config.cookbook_s3_bucket}/#{remote_artifact_file}"
+        })
+      say('Cookbooks updated', :green)
+    else
+      say('You said no, so we\'re done here.', :yellow)
+      exit(-1)
+    end
+  end
+
+  desc 'cleanup', 'Cleanup all temporary files from the cookbook directory'
+  def cleanup
+    Dir.chdir(config.cookbook_dir) do
+      remove_zip_files
+      FileUtils.remove_dir('vendor') if File.directory?('vendor')
+      say("Cleaned up directory '#{config.cookbook_dir}/vendor'", :green)
+    end
+  end
+
+  private
+
+  def initialize_run
+    super
+    config
+  end
+
+  def config
+    return @_config if @_config
+
+    @_config ||= Qops::Environment.new
+
+    %w(cookbook_dir cookbook_s3_bucket cookbook_s3_path cookbook_name cookbook_version).each do |var|
+      fail ArgumentError.new("Must specify a '#{var}' in the config") if !@_config.respond_to?(var) && !@_config.configuration.respond_to?(var)
+    end
+
+    fail ArgumentError.new("Cannot find/do not have access to cookbook directory: #{@_config.cookbook_dir}") unless Dir.exist?(@_config.cookbook_dir)
+
+    @_config
+  end
+
+  def s3
+    @s3 ||= Aws::S3::Client.new(
+      region: 'us-east-1',
+      access_key_id: config.opsworks.config.credentials.access_key_id,
+      secret_access_key: config.opsworks.config.credentials.secret_access_key
+    )
+  end
+
+  def local_artifact_file
+    File.join(config.cookbook_dir, artifact_name)
+  end
+
+  def remote_artifact_file
+    File.join(config.cookbook_s3_path, artifact_name)
+  end
+
+  def artifact_name
+    "#{config.cookbook_name}-#{config.cookbook_version}.zip"
+  end
+
+  def vendor_dir
+    File.join(config.cookbook_dir, 'vendor')
+  end
+
+  def remove_zip_files
+    FileUtils.rm Dir.glob("#{config.cookbook_name}*.zip")
+    say("Cleaned up directory '#{config.cookbook_dir}/*.zip'", :green)
+  end
+end

data/lib/qops/deployment/app.rb

@@ -0,0 +1,117 @@
+class Qops::Deploy < Thor
+  include Qops::DeployHelpers
+
+  class_option :custom_json, type: :string, aliases: '-j', desc: 'A custom json that will be used during a deployment of the app. ex: \'{ "custom_attrs": "are awesome!"}\''
+
+  desc 'app', 'Deploy the latest version of the app'
+  def app
+    initialize_run
+
+    instances = if config.deploy_type == 'staging'
+                  [retrieve_instance].compact
+                else
+                  retrieve_instances
+                end
+    online_instances = instances.select { |instance| instance.status == 'online' }
+
+    if online_instances.count == 0
+      raise 'Could not find any running instance(s) to deploy to. Perhaps you need to run "qops:instance:up" first'
+    end
+
+    if config.deploy_type == 'staging'
+      puts "Preparing to deploy branch #{default_revision} to instance #{online_instances.first.hostname}"
+    else
+      puts "Preparing to deploy default branch to all (online) servers (#{online_instances.map(&:hostname).join(', ')})"
+    end
+
+    base_deployment_params = {
+      stack_id: config.stack_id,
+      command: { name: 'deploy' }
+    }
+
+    if !config.application_id
+      Qops::Environment.print_with_colour('No application specified. Exiting without application deployment.')
+      exit(0)
+    else
+      base_deployment_params[:app_id] = config.application_id
+    end
+
+    if config.deploy_type != 'production'
+      base_deployment_params[:custom_json] = custom_json.to_json
+    end
+
+    manifest = { environment: config.deploy_type }
+
+    # Deploy the first instance with migration on
+    first_instance = online_instances.first
+    print "Migrating and deploying first instance (#{first_instance.hostname}) ..."
+    deployment_params = base_deployment_params.deep_dup
+    should_migrate = !config.option?(:migrate) || config.option?(:migrate) && config.migrate == true
+    deployment_params[:command][:args] = { migrate: ['true'] } if should_migrate
+    run_opsworks_command(deployment_params, [first_instance.instance_id])
+    ping_slack(
+      'Quandl::Slack::Release',
+      "Deployed and migrated instance '#{first_instance.hostname}'",
+      'success',
+      manifest.merge(
+        app_name: config.app_name,
+        command: 'deploy + migrate',
+        migrate: should_migrate.to_s,
+        completed: Time.now,
+        hostname: first_instance.hostname,
+        instance_id: first_instance.instance_id
+      )
+    )
+
+    # Deploy any remaining instances with migration off for production
+    return unless config.deploy_type == 'production' && online_instances.count > 1
+
+    print 'Deploying remaining instances ...'
+    deployment_params = base_deployment_params.deep_dup
+    run_opsworks_command(deployment_params)
+    ping_slack(
+      'Quandl::Slack::Release',
+      'Deployed All Instances',
+      'success',
+      manifest.merge(
+        app_name: config.app_name,
+        command: 'deploy',
+        migrate: false,
+        completed: Time.now,
+        hostname: online_instances.map(&:hostname),
+        instance_id: online_instances.map(&:instance_id)
+      )
+    )
+  end
+
+  private
+
+  def custom_json
+    return @_custom_json if @_custom_json
+
+    @_custom_json = {}
+
+    if config.application_id
+      application_name = config.opsworks.describe_apps(app_ids: [config.application_id]).apps.first.name
+
+      @_custom_json[:deploy] = {
+        application_name => {
+          scm: {
+            revision: default_revision
+          }
+        }
+      }
+    end
+
+    if options[:custom_json].present?
+      your_json = JSON.parse(options[:custom_json])
+      @_custom_json.merge!(your_json)
+      puts "Using custom json:\n#{JSON.pretty_generate(@_custom_json)}"
+    end
+
+    @_custom_json
+  rescue JSON::ParserError
+    say('Your custom json has invalid syntax. Failing ...', :red)
+    exit(-1)
+  end
+end

data/lib/qops/deployment/helpers.rb

@@ -0,0 +1,74 @@
+module Qops::DeployHelpers
+  extend ActiveSupport::Concern
+
+  include Qops::Helpers
+
+  included do
+    class_option :branch, type: :string, aliases: '-b', desc: 'The branch to use when deploying to staging type environments'
+  end
+
+  private
+
+  def config
+    return @_config if @_config
+
+    Qops::Environment.notifiers
+    @_config ||= Qops::Environment.new
+
+    fail "Invalid configure deploy_type detected: #{@_config.deploy_type}" unless %w(staging production).include?(@_config.deploy_type)
+
+    @_config
+  end
+
+  def retrieve_instances(options = {})
+    # Describe and create instances as necessary
+    instances_results = config.opsworks.describe_instances({ layer_id: config.layer_id }.merge(options))
+
+    # Determine if instance exists.
+    instances_results.data.instances
+  end
+
+  def retrieve_instance(instance_id = nil)
+    # Retrieve a specific instance as necessary
+    if instance_id
+      instances_results = config.opsworks.describe_instances(instance_ids: [instance_id])
+      return instances_results.data.instances.first
+    end
+
+    # Get instance based on hostname
+    instances_results = config.opsworks.describe_instances(layer_id: config.layer_id)
+
+    # Determine if instance exists.
+    instances = instances_results.data.instances
+
+    return unless instances.map(&:hostname).include?(requested_hostname)
+
+    instances.find { |k| k.hostname == requested_hostname }
+  end
+
+  def requested_hostname
+    return @requested_hostname if @requested_hostname
+    if config.deploy_type == 'staging'
+      @requested_hostname = default_revision.parameterize
+    elsif config.deploy_type == 'production'
+      @requested_hostname = config.app_name
+      existing_hostnames = retrieve_instances.map(&:hostname)
+      @requested_hostname += "-#{existing_hostnames.sort.last.to_s.split('-').last.to_i + 1}"
+    end
+
+    @requested_hostname = @requested_hostname.gsub(/[^A-Za-z0-9\-]+/, '-').gsub(/-+/, '-')
+    @requested_hostname = @requested_hostname[0..62]
+    @requested_hostname = @requested_hostname.match(/^([A-Za-z0-9\-]+).*$/)[1]
+  end
+
+  def default_revision
+    return 'master' unless config.deploy_type == 'staging'
+    if options[:branch].present?
+      options[:branch]
+    elsif `git --version` # rubocop:disable Lint/LiteralInCondition
+      `git symbolic-ref --short HEAD`.strip
+    else
+      'master'
+    end
+  end
+end

data/lib/qops/deployment/instances.rb

@@ -0,0 +1,351 @@
+class Qops::Instance < Thor # rubocop:disable Metrics/ClassLength
+  include Qops::DeployHelpers
+
+  desc 'up', 'Deploy the current branch to new or existing instance(s)'
+  def up
+    initialize_run
+
+    # Get the instance(s) to work with if they exist. In production we always create a new instacne
+    instance = retrieve_instance if config.deploy_type == 'staging'
+
+    # Create the instance if necessary
+    if instance
+      instance_id = instance.instance_id
+      puts "Existing instance #{requested_hostname}"
+    else
+      params = {
+        stack_id: config.stack_id,
+        layer_ids: [config.layer_id],
+        instance_type: config.instance_type,
+        os: 'Ubuntu 14.04 LTS',
+        hostname: requested_hostname,
+        subnet_id: config.subnet,
+        auto_scaling_type: config.autoscale_type,
+        architecture: 'x86_64',
+        root_device_type: 'ebs',
+        block_device_mappings: [
+          {
+            device_name: 'ROOT_DEVICE',
+            ebs: {
+              volume_size: config.root_volume_size,
+              volume_type: 'gp2',
+              delete_on_termination: true
+            }
+          }
+        ],
+        ebs_optimized: config.deploy_type =~ /production/
+      }
+      puts 'Creating instance with params: ' + params.inspect
+      instance_id = config.opsworks.create_instance(params).data.instance_id
+      creating_instance = true
+    end
+
+    instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
+    instance = instance_results.data.instances.first
+
+    # Set up the automatic boot scheduler
+    if config.autoscale_type == 'timer'
+      print 'Setting up weekly schedule ...'
+      config.opsworks.set_time_based_auto_scaling(instance_id: instance_id, auto_scaling_schedule: config.schedule)
+      print "done\n"
+    end
+
+    # Record the initial instance before doing anything.
+    initial_instance_state = instance
+
+    # Start the instance if necessary
+    print 'Booting instance ...'
+    unless %w(online booting).include?(instance.status)
+      config.opsworks.start_instance(instance_id: instance_id)
+    end
+
+    manifest = {
+      environment: config.deploy_type,
+      app_name: config.app_name,
+      command: 'add instance'
+    }
+
+    # Boot the instance
+    iterator(manifest) do |i|
+      instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
+      instance = instance_results.data.instances.first
+
+      if %w(booting requested pending).include?(instance.status)
+        print '.'
+        print " #{instance.status} :" if been_a_minute?(i)
+      else
+        puts ' ' + instance.status
+        true
+      end
+    end
+
+    puts "Public IP: #{instance.public_ip}"
+    puts "Private IP: #{instance.private_ip}"
+
+    setup_instance(instance, initial_instance_state, manifest)
+
+    if creating_instance
+      ping_slack(
+        'Quandl::Slack::InstanceUp',
+        'Created another instance',
+        'success',
+        manifest.merge(
+          completed: Time.now,
+          hostname: instance.hostname,
+          instance_id: instance.instance_id,
+          private_ip: instance.private_ip,
+          public_ip: instance.public_ip.blank? ? 'N/A' : instance.public_ip
+        )
+      )
+    end
+
+    # For Elasticsearch cluster, register with public elb
+    if config.option?(:public_search_elb)
+      print "Register instance #{instance.ec2_instance_id} to elb #{config.public_search_elb}"
+      elb.register_instances_with_load_balancer(load_balancer_name: config.public_search_elb.to_s,
+                                                instances: [{ instance_id: instance.ec2_instance_id.to_s }])
+    end
+
+    # Deploy the latest code to instance
+    Qops::Deploy.new([], options).app
+  end
+
+  desc 'down', 'Remove the instance associated with the given branch'
+  def down
+    initialize_run
+
+    # Get the instance to shutdown
+    if config.deploy_type == 'staging'
+      instance = retrieve_instance
+    elsif config.deploy_type == 'production'
+      instance = retrieve_instances.first
+    end
+
+    if instance.nil?
+      puts 'No instance available to shutdown'
+      exit(0)
+    else
+      instance_id = instance.instance_id
+    end
+
+    terminate_instance(instance_id)
+
+    puts 'Success'
+  end
+
+  desc 'rebuild', 'Runs the down then up command to rebuild an instance.'
+  def rebuild
+    down
+    up
+  end
+
+  desc 'clean', 'Cleans up old instances in staging type environments'
+  def clean
+    initialize_run
+
+    if config.deploy_type == 'production'
+      fail "Cannot clean instances in a #{config.deploy_type} environment"
+    end
+
+    terminated_instances = []
+
+    # Find all instances to be destroyed
+    retrieve_instances.each do |instance|
+      next if instance.hostname == 'master'
+
+      # Find the latest command since the instance was deployed
+      latest_command = Time.parse(instance.created_at)
+      config.opsworks.describe_commands(instance_id: instance.instance_id).commands.each do |command|
+        next if config.clean_commands_to_ignore.include?(command.type)
+        completed_at = Time.parse(command.completed_at || command.acknowledged_at || command.created_at)
+        latest_command = completed_at if completed_at > latest_command
+      end
+
+      # If the latest deployment is greater than the maximum alive time allowed remove the instance.
+      if Time.now.to_i - latest_command.to_i > config.max_instance_duration
+        terminate_instance(instance.instance_id)
+        terminated_instances << instance
+      end
+    end
+
+    if terminated_instances.any?
+      puts "Terminated instances: #{terminated_instances.map(&:hostname).join("\n")}"
+    else
+      puts 'No unused instances old enough to terminate.'
+    end
+  end
+
+  desc 'run_command', 'Run command on existing instance(s) at once or each one by one'
+  def run_command
+    initialize_run
+    instances = retrieve_instances
+
+    puts "Preparing to run command to all servers (#{instances.map(&:hostname).join(', ')})"
+
+    command = ask('Which command you want to execute?', limited_to: %w(setup configure install_dependencies update_dependencies))
+
+    option = ask('Which command you want to execute?', limited_to: %w(all_in_once one_by_one))
+
+    base_deployment_params = {
+      stack_id: config.stack_id,
+      command: { name: command.to_s }
+    }
+
+    manifest = { environment: config.deploy_type }
+
+    case option
+    when 'all_in_once'
+      print "Run command #{command} on all instances at once ..."
+      deployment_params = base_deployment_params.deep_dup
+      run_opsworks_command(deployment_params)
+      ping_slack(
+        'Quandl::Slack::Release',
+        "Run command: `#{command}` on all instances",
+        'success',
+        manifest.merge(
+          app_name: config.app_name,
+          command: 'deploy',
+          migrate: false,
+          completed: Time.now,
+          hostname: instances.map(&:hostname),
+          instance_id: instances.map(&:instance_id)
+        )
+      )
+    else
+      instances.each do |instance|
+        print "Run command #{command} on instance #{instance.ec2_instance_id}"
+
+        run_opsworks_command(base_deployment_params, [instance.instance_id])
+
+        ping_slack('Quandl::Slack::InstanceDown', "Run command: `#{command}` on existing instance", 'success',
+                   manifest.merge(
+                     completed: Time.now,
+                     hostname: instance.hostname,
+                     instance_id: instance.instance_id,
+                     private_ip: instance.private_ip,
+                     public_ip: instance.public_ip
+                   )
+                  )
+        puts 'Success'
+        break if instance.instance_id == instances.last.instance_id
+        delay = config.wait_deploy
+        puts "wait for #{delay / 60.0} mintues"
+        sleep delay
+      end
+    end
+  end
+
+  private
+
+  def elb
+    @elb ||= Aws::ElasticLoadBalancing::Client.new(
+      region: 'us-east-1',
+      access_key_id: config.opsworks.config.credentials.access_key_id,
+      secret_access_key: config.opsworks.config.credentials.secret_access_key)
+  end
+
+  def setup_instance(instance, initial_instance_state, manifest)
+    # If the previous instance setup failed then run the setup task again when trying to bring up the instance.
+    if initial_instance_state.status == 'setup_failed'
+      print 'Setup instance ...'
+      run_opsworks_command(
+        {
+          stack_id: config.stack_id,
+          command: {
+            name: 'setup'
+          }
+        },
+        [instance.instance_id]
+      )
+
+    # Monitor the existing instance setup.
+    else
+      print 'Setup instance ...'
+      iterator(manifest) do |i|
+        instance_results = config.opsworks.describe_instances(instance_ids: [instance.instance_id])
+        instance = instance_results.data.instances.first
+
+        if %w(online).include?(instance.status)
+          puts ' ' + instance.status
+          true
+        elsif %w(setup_failed).include?(instance.status)
+          puts ' ' + instance.status
+          read_failure_log(
+            { instance_id: instance.instance_id },
+            last_only: true,
+            manifest: manifest.merge(
+              hostname: instance.hostname,
+              instance_id: instance.instance_id,
+              private_ip: instance.private_ip,
+              public_ip: instance.public_ip
+            )
+          )
+          exit(-1)
+        else
+          print '.'
+          print " #{instance.status} :" if been_a_minute?(i)
+        end
+      end
+    end
+  end
+
+  def terminate_instance(instance_id)
+    # Remove schedule if time based instance
+    if config.autoscale_type == 'timer'
+      config.opsworks.set_time_based_auto_scaling(instance_id: instance_id, auto_scaling_schedule: {})
+    end
+
+    # Get the instance from the id
+    instance = retrieve_instance(instance_id)
+
+    # For Elasticsearch cluster, remove from from public elb
+    if config.option?(:public_search_elb)
+      elb.deregister_instances_from_load_balancer(
+        load_balancer_name: config.public_search_elb.to_s,
+        instances: [{ instance_id: instance.ec2_instance_id.to_s }]
+      )
+    end
+
+    # Attempt to shutdown the instance
+    print "Attempting instance #{instance_id}-#{instance.hostname} shutdown ..."
+    unless instance.status == 'stopped'
+      config.opsworks.stop_instance(instance_id: instance_id)
+    end
+
+    manifest = {
+      environment: config.deploy_type,
+      app_name: config.app_name,
+      command: 'remove instance'
+    }
+
+    iterator(manifest) do |i|
+      instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
+      instance = instance_results.data.instances.first
+
+      if instance.status == 'stopped'
+        puts ' ' + instance.status
+        true
+      else
+        print '.'
+        print " #{instance.status} :" if been_a_minute?(i)
+      end
+    end
+
+    # Terminate the instance
+    puts "Terminating instance #{instance_id}"
+    config.opsworks.delete_instance(instance_id: instance_id, delete_volumes: true)
+
+    ping_slack(
+      'Quandl::Slack::InstanceDown',
+        'Remove existing instance',
+        'success',
+        manifest.merge(
+          completed: Time.now,
+          hostname: instance.hostname,
+          instance_id: instance.instance_id,
+          private_ip: instance.private_ip,
+          public_ip: instance.public_ip
+        )
+    )
+  end
+end

data/lib/qops/environment.rb

@@ -0,0 +1,110 @@
+require 'quandl/config'
+
+module Qops
+  class Environment
+    include Quandl::Configurable
+
+    def self.file_name
+      'opsworks'
+    end
+
+    def self.notifiers
+      return @_notifiers unless @_notifiers.nil?
+      if File.exist?('config/quandl/slack.yml')
+        @_notifiers ||= Quandl::Slack.autogenerate_notifiers
+      else
+        @_notifiers = false
+        print_with_colour('Slack notifications disabled. Could not find slack configuration at: config/quandl/slack.yml', :warning)
+      end
+    rescue NoMethodError => e
+      print_with_colour("Slack notifications disabled due to an error. #{e}", :warning)
+    end
+
+    def self.print_with_colour(message, level = :normal)
+      case level
+      when :error
+        puts Rainbow(message).bg(:black).red
+      when :warning
+        puts Rainbow(message).bg(:black).yellow
+      else
+        puts message
+      end
+    end
+
+    def initialize
+      %w(deploy_type region stack_id app_name).each do |v|
+        fail "Please configure #{v} before continuing." unless option?(v)
+      end
+
+      begin
+        opsworks.config.credentials.access_key_id
+        opsworks.config.credentials.secret_access_key
+      rescue => e
+        raise "There may be a problem with your aws credentials. Please correct with `aws configure`. Error: #{e}"
+      end
+    end
+
+    def application_id
+      configuration.application_id unless configuration.application_id.blank?
+    end
+
+    def deploy_type
+      configuration.deploy_type.to_s
+    end
+
+    def command_log_lines
+      configuration.command_log_lines || 100
+    end
+
+    def wait_iterations
+      configuration.wait_iterations || 600
+    end
+
+    def wait_deploy
+      configuration.wait_deploy || 180
+    end
+
+    def autoscale_type
+      configuration.autoscale_type || nil
+    end
+
+    # Default 1 days
+    def max_instance_duration
+      configuration.max_instance_duration || 86_400
+    end
+
+    def clean_commands_to_ignore
+      configuration.clean_commands_to_ignore.present? ? configuration.clean_commands_to_ignore : %w(configure shutdown)
+    end
+
+    def file_name
+      self.class.file_name
+    end
+
+    def opsworks
+      @_opsworks ||= Aws::OpsWorks::Client.new(region: configuration.region)
+    end
+
+    def cookbook_json
+      configuration.cookbook_json || 'custom.json'
+    end
+
+    def option?(key)
+      respond_to?(key.to_sym) || configuration.instance_variable_get(:@table).keys.include?(key.to_sym)
+    end
+
+    def root_volume_size
+      configuration.root_volume_size || 30
+    end
+
+    private
+
+    def method_missing(method_sym, *arguments, &block)
+      if configuration.respond_to?(method_sym)
+        configuration.send(method_sym, *arguments, &block)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/qops/helpers.rb

@@ -0,0 +1,153 @@
+module Qops::Helpers
+  extend ActiveSupport::Concern
+
+  included do
+    class_option :environment, aliases: '-e', desc: 'The environment to use when running commands.'
+  end
+
+  private
+
+  def initialize_run
+    return if @_run_initialized
+
+    @_run_initialized = true
+    verify_opsworks_config
+    verify_environment_selection
+    Qops::Environment.notifiers
+  end
+
+  def been_a_minute?(i)
+    i > 1 && i % 60 == 0
+  end
+
+  def iterator(options)
+    config.wait_iterations.times.each do |i|
+      result = yield(i)
+      break if result
+
+      if i + 1 == config.wait_iterations
+        puts " failed to complete within #{config.wait_iterations} seconds"
+        ping_slack('Quandl::Slack::Release', 'Command timeout', 'failure',
+                   options.merge(failed_at: Time.now)
+                  )
+        exit(-1)
+      elsif been_a_minute?(i)
+        print " #{i / 60} minute(s) "
+      end
+
+      sleep(1)
+    end
+  end
+
+  def ping_slack(notifier, message, status, manifest)
+    fields = manifest.keys
+    fields.map! { |field| { title: field, value: manifest[field], short: true } }
+
+    if Object.const_defined?(notifier)
+      notifier_class = notifier.constantize
+
+      notifier_class.ping("#{config.app_name}: #{message}",
+                          attachments: [{ color: status, mrkdwn_in: ['text'], fallback: 'Details', fields: fields }]
+                         )
+    else
+      puts "#{config.app_name}: #{message}"
+      pp fields
+    end
+  end
+
+  def run_opsworks_command(deployment_params, instance_ids = [])
+    deployment_params[:instance_ids] = instance_ids unless instance_ids.empty?
+
+    # Create the deployment
+    deployment_results = config.opsworks.create_deployment(deployment_params)
+    deployment_id = deployment_results.data.deployment_id
+
+    iterator(deployment_params) do |i|
+      deployment_results = config.opsworks.describe_deployments(deployment_ids: [deployment_id])
+      deployment = deployment_results.data.deployments.first
+
+      if deployment.completed_at
+        puts ' ' + deployment.status
+
+        if deployment.status != 'successful'
+          read_failure_log(deployment_id: deployment.deployment_id)
+          exit(-1)
+        end
+
+        true
+      else
+        print '.'
+        if been_a_minute?(i)
+          print " #{retrieve_instance.status} :" if config.deploy_type == :staging && instance_ids.any?
+          print " #{deployment.status} :" if config.deploy_type == :production
+        end
+      end
+    end
+
+    print "\n"
+  end
+
+  def read_failure_log(opsworks_options, options = {})
+    results = config.opsworks.describe_commands(opsworks_options)
+    results.commands.each do |command|
+      if command.log_url
+        puts "\nReading last 100 lines from #{command.log_url}\n"
+        lines = open(command.log_url).read.split("\n")
+        num_lines = lines.count < config.command_log_lines ? lines.count : config.command_log_lines
+        puts open(command.log_url).read.split("\n")[-1 * num_lines..-1].join("\n")
+        puts "\nLog file at: #{command.log_url}"
+      end
+
+      ping_slack(
+        'Quandl::Slack::Release',
+        'Deployment failure',
+        'failure',
+        (options[:manifest] || {}).merge(
+          command: command.type,
+          status: command.status
+        )
+      )
+
+      exit(-1) if options[:last_only]
+    end
+
+    exit(-1)
+  end
+
+  def verify_opsworks_config
+    return if File.exist?("config/#{Qops::Environment.file_name}.yml")
+    raise "Could not find configuration file: config/#{Qops::Environment.file_name}.yml"
+  end
+
+  def verify_environment_selection
+    return if Quandl::Config.environment
+
+    project_root = Pathname.new(Quandl::ProjectRoot.root)
+    file_path = project_root.join('config', "#{Qops::Environment.file_name}.yml")
+
+    if File.exist?(file_path) # rubocop:disable Style/GuardClause
+      raw_config = File.read(file_path)
+      erb_config = ERB.new(raw_config).result
+      configs = YAML.load(erb_config)
+
+      env = options[:environment]
+
+      msg = 'Run command using config environment:'
+      msg = "Invalid config environment '#{env}'. Switch to:" if env && !configs.keys.include?(env)
+
+      unless env && configs.keys.include?(env)
+        env = Thor::Shell::Color.new.ask(
+          msg,
+          :yellow,
+          limited_to: configs.keys.reject { |g| g.start_with?('_') },
+          echo: false
+        )
+      end
+
+      Quandl::Config.environment = env
+      puts "\nRunning commands with config environment: #{env}"
+    else
+      raise "Not a qops compatible project. Please be sure to add a config/opsworks.yml file as described in the readme. Path: #{file_path}"
+    end
+  end
+end

data/lib/qops/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Qops
+  VERSION = '1.0.0'.freeze
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: qops
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Matthew Basset
+- Clemeny Leung
+- Jason Byck
+- Jun Li
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.41
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.41
+- !ruby/object:Gem::Dependency
+  name: quandl-config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: quandl-slack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.1
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+description: Help to automate opsworks project deployments with single commands.
+email:
+- support@quandl.com
+executables:
+- qops
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/qops
+- lib/qops.rb
+- lib/qops/cookbook/cookbook.rb
+- lib/qops/deployment/app.rb
+- lib/qops/deployment/helpers.rb
+- lib/qops/deployment/instances.rb
+- lib/qops/environment.rb
+- lib/qops/helpers.rb
+- lib/qops/version.rb
+homepage: https://github.com/quandl/opsworks_commands
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.4
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Helper commands for deployment of opsworks projects.
+test_files: []
+has_rdoc: