zzdeploy 0.0.5

data/LICENSE

@@ -0,0 +1 @@
+Copyright (c) 2011 ZangZing, LLC

data/README.rdoc

@@ -0,0 +1,10 @@
+= zz
+
+Deploy tool for ZangZing and Amazon instances
+
+== Installation
+  $ gem install zzdeploy
+
+== Copyright
+
+Copyright (c) 2011 ZangZing, LLC. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,25 @@
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  require 'zz_deploy'
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "zzdeploy #{ZZDeploy::VERSION}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => :spec

data/bin/zz

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# ./zz - ZZ deploy
+#
+
+require 'rubygems'
+require "bundler/setup"
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'zz_deploy'
+
+exit_code = ZZDeploy.new.run
+exit!(exit_code)
+
+

data/lib/commands/add_instance.rb

@@ -0,0 +1,187 @@
+module Commands
+  class AddInstance
+
+    # holds the options that were passed
+    # you can set any initial defaults here
+    def options
+      @options ||= {
+          :instance_size => "c1.medium",
+          :start_app => false,
+          :availability_zone => "us-east-1c"   # this is where our reserved instance currently are
+      }
+    end
+
+    # required options
+    def required_options
+      @required_options ||= Set.new [
+          :role,
+          :group,
+      ]
+    end
+
+    def register(opts, global_options)
+      opts.banner = "Usage: add [options]"
+      opts.description = "Add a server instance"
+
+      opts.on('-s', "--size instance_size", ["t1.micro", "m1.small", "c1.medium"], "The amazon instance size - currently we limit to 32 bit instances.") do |v|
+        options[:instance_size] = v
+      end
+
+      opts.on('-z', "--zone availability_zone", MetaOptions.availability_zones, "The amazon availability zone - currently we only support the east coast.") do |v|
+        options[:availability_zone] = v
+      end
+
+      opts.on('-r', "--role role", MetaOptions.roles, "Required - Role server will play.") do |v|
+        options[:role] = v
+      end
+
+      opts.on("--start_app", "Set if you want to deploy and start the app after instance is ready.") do |v|
+        options[:start_app] = v
+      end
+
+      opts.on('-e', "--extra extra", MetaOptions.roles, "Optional extra data associated with this instance.") do |v|
+        options[:extra] = v
+      end
+
+      opts.on('-g', "--group deploy_group", "Required - The deploy group we are in.  A deploy group is a set of servers that are required to run the infrastructure for a server.") do |v|
+        options[:group] = v
+      end
+
+      opts.on('-p', "--print path", "The directory into which we output the data as a file per host.") do |v|
+        options[:result_path] = v
+      end
+    end
+
+
+    def run(global_options, amazon)
+      ec2 = amazon.ec2
+      utils = ZZSharedLib::Utils.new(amazon)
+
+      user_json = JSON.pretty_generate(options)
+
+      # deploy group
+      group_name = options[:group]
+      role = options[:role]
+      extra = options[:extra]
+      start_app = options[:start_app]
+
+      # first see if already exists
+      deploy_group = amazon.find_deploy_group(group_name)
+      recipes_deploy_tag = deploy_group.recipes_deploy_tag
+      group_config = deploy_group.config
+
+      availability_zone = options[:availability_zone] || group_config[:availability_zone]
+
+      # the security key
+      security_key = group_config[:amazon_security_key]
+
+      # the security group
+      security_group = group_config[:amazon_security_group]
+
+      # find ones matching the role
+      match = amazon.find_by_role(group_name, role)
+
+      # stop if we have a case where we only allow one of a particular kind
+      case role.to_sym
+        when :app_master, :db, :solo
+          if match.length > 0
+            raise "Argument error: You already have a duplicate role of #{role}.  The existing instance is #{match[0]}."
+          end
+      end
+
+      # now find the proper AMI image to use
+      baseline_image = group_config[:amazon_image]
+
+      # first see if we have a specific image for this role
+      role_image = "#{baseline_image}_#{role}"
+      match_image = amazon.find_typed_resource("image", "Name", role_image)
+      if match_image.length > 1
+        raise "You must have only one AMI Image for #{role_image}.  Found: #{match_image.length}"
+      end
+
+      if match_image.length != 1
+        # didn't have a specific role image so get the generic one
+        match_image = amazon.find_typed_resource("image", "Name", baseline_image)
+        if match_image.length != 1
+          raise "Need to have exactly one AMI Image for #{baseline_image}.  Found: #{match_image.length}"
+        end
+      end
+
+
+      instances = ec2.run_instances(match_image[0], 1, 1, [security_group], security_key, user_json, nil, options[:instance_size],
+                                    nil, nil, availability_zone)
+      inst_id = instances[0][:aws_instance_id]
+      puts "Waiting for instance #{inst_id} to boot"
+      aws_state = ""
+      instance = nil
+      waits = 0
+      while aws_state != "running" do
+        print "."
+        STDOUT.flush
+        sleep(1)
+        waits += 1
+        # this silly bit of logic is needed because sometimes Amazons API does not know
+        # about the newly created instance for a brief period so don't ask till we give it
+        # a chance to learn about it
+        if waits >= 10
+          instance = ec2.describe_instances(inst_id)[0]
+          aws_state = instance[:aws_state]
+        end
+      end
+      puts
+      puts "Tagging instance."
+      ec2.create_tags(inst_id, {"Name" => "#{group_name}_#{role}_#{inst_id}", :group => group_name, :role => role, :extra => extra,
+                                :state => 'booting', :deploy_app => ZZSharedLib::Utils::NEVER, :deploy_chef => ZZSharedLib::Utils::NEVER })
+
+      dns_name = instance[:dns_name]
+      ssh_cmd = "ssh -t -i ~/.ssh/#{group_config[:amazon_security_key]}.pem ec2-user@#{dns_name}"
+      puts ssh_cmd
+      test_cmd = 'echo "connected"'
+      test_cmd = "#{ssh_cmd} '#{test_cmd}'"
+      tries = 0
+      while true do
+        puts "testing ssh connection"
+        result = ZZSharedLib::CL.do_cmd_result test_cmd
+        break if result == 0
+        sleep(6)
+        tries += 1
+        if tries >= 10
+          # todo decide if we should terminate this instance
+          ec2.create_tags(inst_id, {:state => 'failed_boot' })
+          raise "Not able to establish ssh connection, make sure the security group has the ssh port open."
+        end
+      end
+
+      # if we get here we should have verified that the machine is ready and we can ssh into it, lets
+      # do the initial upload step for the chef recipes by fetching the proper tag on the remote machine
+      ec2.create_tags(inst_id, {:state => 'ready' })
+
+      git_cmd = ChefUpload.get_upload_command(recipes_deploy_tag)
+      remote_cmd = "#{ssh_cmd} \"#{git_cmd}\""
+      result = ZZSharedLib::CL.do_cmd_result remote_cmd
+      if result != 0
+        raise "The instance was created but we were unable to upload the chef recipes.\nYou should try again by using 'chef_upload' and make sure you have a valid git tag."
+      end
+
+      # set up our instance id
+      # first get all instances in our group which should include us
+      amazon.flush_tags    # force a refresh of the cached tags
+      all_instances = amazon.find_and_sort_named_instances(group_name)
+
+      just_our_instance = all_instances.reject { |inst| inst[:resource_id] != inst_id }
+
+      # deploy the chef config
+      puts "Updating chef configuration for new instance."
+      BuildDeployConfig.do_config_deploy(utils, amazon, just_our_instance, group_name, deploy_group, options[:result_path])
+
+      # optionally deploy and start app
+      # note in this case we redeploy the whole group since
+      # there are dependencies between the instances
+      if start_app
+        puts "Now deploying all app instances since the configuration changed."
+        BuildDeployConfig.do_app_deploy(utils, amazon, all_instances, group_name, deploy_group, '', false, false, options[:result_path])
+      end
+
+    end
+  end
+end

data/lib/commands/build_deploy_config.rb

@@ -0,0 +1,143 @@
+module Commands
+
+  # this class builds up the contents of the json deploy file that we feed
+  # to the remote system to kickstart the chef run.  It allows us to
+  # dynamically configure the options we want
+  class BuildDeployConfig
+
+    def self.build_json(zz_options, deploy_file)
+      chef_config = { :run_list => "recipe[deploy-manager]" }
+      zz_options[:dev_machine] = false
+      chef_config[:zz] = zz_options
+
+      json = JSON.pretty_generate(chef_config)
+
+      # now build up the command string to run
+      cmd =
+"bash -l -c '(
+cat <<'EOP'
+#{json}
+EOP
+) > /var/chef/#{deploy_file}.json
+cd #{::ZZDeploy::RECIPES_DIR}
+sudo bundle exec chef-solo -l debug -c chef-local/remote_solo.rb -j /var/chef/#{deploy_file}.json'"
+
+      return cmd
+    end
+
+    def self.remote_app_deploy(migrate_command, downtime)
+      zz = {
+          :deploy_what => "app",
+          :deploy_migrate_command => migrate_command,
+          :deploy_downtime => downtime
+      }
+      return build_json(zz, 'deploy_app')
+    end
+
+    def self.remote_app_restart(migrate_command, downtime)
+      zz = {
+          :deploy_what => "app_restart",
+          :deploy_migrate_command => migrate_command,
+          :deploy_downtime => downtime
+      }
+      return build_json(zz, 'deploy_app_restart')
+    end
+
+    def self.remote_app_maint(maint)
+      zz = {
+          :deploy_what => "app_maint",
+          :deploy_maint => maint
+      }
+      return build_json(zz, 'deploy_app_maint')
+    end
+
+    def self.remote_config_deploy
+      zz = {
+          :deploy_what => "config"
+      }
+      return build_json(zz, 'deploy_config')
+    end
+
+    def self.remote_shutdown
+      zz = {
+          :deploy_what => "shutdown"
+      }
+      return build_json(zz, 'deploy_shutdown')
+    end
+
+    def self.do_remote_shutdown(utils, amazon, instances, group_name, deploy_group, result_path)
+      begin
+        remote_cmd = remote_shutdown
+        multi = MultiSSH.new(amazon, group_name, deploy_group)
+        multi.run_instances(instances, remote_cmd)
+      rescue Exception => ex
+        # ignore any errors since we want to shut down regardless
+      ensure
+        multi.output_tracked_data_to_files("remote_shutdown", result_path) rescue nil
+      end
+    end
+
+    def self.do_config_deploy(utils, amazon, instances, group_name, deploy_group, result_path)
+      begin
+        utils.mark_deploy_state(instances, :deploy_chef, ZZSharedLib::Utils::START)
+        remote_cmd = remote_config_deploy
+        multi = MultiSSH.new(amazon, group_name, deploy_group)
+        multi.run_instances(instances, remote_cmd)
+      rescue Exception => ex
+        raise ex
+      ensure
+        multi.output_tracked_data_to_files("deploy_chef", result_path) rescue nil
+        # only mark ready if not in error state
+        utils.mark_deploy_state(instances, :deploy_chef, ZZSharedLib::Utils::READY, true)
+      end
+    end
+
+    def self.do_app_deploy(utils, amazon, instances, group_name, deploy_group, migrate_command, downtime, no_restart, result_path)
+      # ok, phase one is to do everything up to but not including the restart
+      begin
+        utils.mark_deploy_state(instances, :deploy_app, ZZSharedLib::Utils::START)
+        remote_cmd = remote_app_deploy(migrate_command, downtime)
+        multi = MultiSSH.new(amazon, group_name, deploy_group)
+        multi.run_instances(instances, remote_cmd)
+      rescue Exception => ex
+        raise ex
+      ensure
+        multi.output_tracked_data_to_files("app_deploy", result_path) rescue nil
+      end
+
+      # the prep is good and all servers are ready to restart
+      begin
+        if no_restart == false
+          puts "Restarting servers..."
+          utils.mark_deploy_state(instances, :deploy_app, ZZSharedLib::Utils::RESTARTING)
+          remote_cmd = remote_app_restart(migrate_command, downtime)
+          multi.run_instances(instances, remote_cmd)
+        end
+      rescue Exception => ex
+        raise ex
+      ensure
+        multi.output_tracked_data_to_files("app_restart", result_path) rescue nil
+        # mark as ready unless they are in the error state already
+        utils.mark_deploy_state(instances, :deploy_app, ZZSharedLib::Utils::READY, true)
+      end
+    end
+
+    # turn off or on the maintenance mode
+    def self.do_maint_deploy(utils, amazon, instances, group_name, deploy_group, maint, result_path)
+      begin
+        utils.mark_deploy_state(instances, :deploy_app, ZZSharedLib::Utils::MAINT)
+        remote_cmd = remote_app_maint(maint)
+        multi = MultiSSH.new(amazon, group_name, deploy_group)
+        multi.run_instances(instances, remote_cmd)
+      rescue Exception => ex
+        raise ex
+      ensure
+        multi.output_tracked_data_to_files("app_maint", result_path) rescue nil
+        # only mark ready if not in error state
+        utils.mark_deploy_state(instances, :deploy_app, ZZSharedLib::Utils::READY, true)
+      end
+    end
+
+  end
+
+end

data/lib/commands/chef_bake.rb

@@ -0,0 +1,61 @@
+module Commands
+  class ChefBake
+
+    # holds the options that were passed
+    # you can set any initial defaults here
+    def options
+      @options ||= {
+      }
+    end
+
+    # required options
+    def required_options
+      @required_options ||= Set.new [
+          :group,
+      ]
+    end
+
+    def register(opts, global_options)
+      opts.banner = "Usage: chef_apply [options]"
+      opts.description = "Apply the chef scripts"
+
+      opts.on('-g', "--group name", "Required - Name of this deploy group.") do |v|
+        options[:group] = v
+      end
+
+      opts.on('-f', "--force", "Force a deploy even if the current status says we are deploying.  You should only do this if you are certain the previous deploy is stuck.") do |v|
+        options[:force] = v
+      end
+    end
+
+
+    def run(global_options, amazon)
+      ec2 = amazon.ec2
+      utils = ZZSharedLib::Utils.new(amazon)
+
+      group_name = options[:group]
+
+      # first see if already exists
+      deploy_group = amazon.find_deploy_group(group_name)
+
+      recipes_deploy_tag = deploy_group.recipes_deploy_tag
+
+      instances = amazon.find_and_sort_named_instances(group_name)
+
+      # see if already deploying
+      if !options[:force]
+        # raises an exception if not all in the none state
+        utils.check_deploy_state(instances, [:deploy_chef, :deploy_app])
+      end
+
+      # verify that the chef deploy tag exists
+      cmd = "git ls-remote --tags git@github.com:zangzing/zz-chef-repo.git refs/tags/#{recipes_deploy_tag} | egrep refs/tags/#{recipes_deploy_tag}"
+      if ZZSharedLib::CL.do_cmd_result(cmd) != 0
+        raise "Could not find the tag: #{recipes_deploy_tag} in the remote zz-chef-repo repository.  Make sure you check in and tag your code, and run chef_upload."
+      end
+
+      # tag is good, go ahead and deploy to all the machines in the group
+      BuildDeployConfig.do_config_deploy(utils, amazon, instances, group_name, deploy_group, options[:result_path])
+    end
+  end
+end

data/lib/commands/chef_upload.rb

@@ -0,0 +1,62 @@
+module Commands
+  class ChefUpload
+
+    # holds the options that were passed
+    # you can set any initial defaults here
+    def options
+      @options ||= {
+      }
+    end
+
+    # required options
+    def required_options
+      @required_options ||= Set.new [
+          :group,
+          :tag,
+      ]
+    end
+
+    def register(opts, global_options)
+      opts.banner = "Usage: chef_upload [options]"
+      opts.description = "Upload the chef scripts"
+
+      opts.on('-g', "--group name", "Required - Name of this deploy group.") do |v|
+        options[:group] = v
+      end
+
+      opts.on('-t', "--tag tag", "Required - Git tag to use for pulling chef code.") do |v|
+        options[:tag] = v
+      end
+    end
+
+
+    def run(global_options, amazon)
+      ec2 = amazon.ec2
+
+      group_name = options[:group]
+      recipes_deploy_tag = options[:tag]
+
+      # first see group exists
+      deploy_group = amazon.find_deploy_group(group_name)
+
+      # verify that the tag given is on the remote repo
+      cmd = "git ls-remote --tags git@github.com:zangzing/zz-chef-repo.git refs/tags/#{recipes_deploy_tag} | egrep refs/tags/#{recipes_deploy_tag}"
+      if ZZSharedLib::CL.do_cmd_result(cmd) != 0
+        raise "Could not find the tag specified in the remote zz-chef-repo repository.  Make sure you check in and tag your code."
+      end
+
+      # tag is good, go ahead and upload to simple db
+      deploy_group.recipes_deploy_tag = recipes_deploy_tag
+      deploy_group.save
+      deploy_group.reload # save corrupts the in memory state so must reload, kinda lame
+
+      remote_cmd = ChefUpload.get_upload_command(recipes_deploy_tag)
+      multi = MultiSSH.new(amazon, group_name, deploy_group)
+      multi.run(remote_cmd)
+    end
+
+    def self.get_upload_command(recipes_deploy_tag)
+      "cd #{::ZZDeploy::RECIPES_DIR} && git fetch && git checkout -f #{recipes_deploy_tag} && git pull && bundle install --path #{::ZZDeploy::RECIPES_BUNDLE_DIR} --deployment"
+    end
+  end
+end

data/lib/commands/config_amazon.rb

@@ -0,0 +1,64 @@
+
+module Commands
+  class ConfigAmazon
+
+    # holds the options that were passed
+    # you can set any initial defaults here
+    def options
+      @options ||= {
+      }
+    end
+
+    # required options
+    def required_options
+      @required_options ||= Set.new [
+          :access_key,
+          :secret_key
+      ]
+    end
+
+    def register(opts, global_options)
+      opts.banner = "Usage: config_amazon [options]"
+      opts.description = "Write the amazon key configuration"
+
+      opts.on("--akey AmazonAccessKey", "Required: Amazon access key, or environment AWS_ACCESS_KEY_ID.") do |v|
+        options[:access_key] = v
+      end
+
+      opts.on("--skey AmazonSecretKey", "Required: Amazon secret key or environment AWS_SECRET_ACCESS_KEY") do |v|
+        options[:secret_key] = v
+      end
+    end
+
+
+    def run(global_options, amazon)
+      ec2 = amazon.ec2
+
+      access_key = options[:access_key]
+      secret_key = options[:secret_key]
+
+      info = {
+          :aws_access_key_id => access_key,
+          :aws_secret_access_key => secret_key
+      }
+
+      # make sure the chef dir exists
+      `sudo mkdir -p #{chef_dir}`
+
+      # generate the json into a temp file
+      json = JSON.pretty_generate(info)
+      chef_dir = "/var/chef"
+      amazon_path = "#{chef_dir}/amazon.json"
+      temp_path = File.expand_path('.', "~/amazon_temp.json")
+      File.open(temp_path, 'w') {|f| f.write(json) }
+
+      # now move the file and set permissions
+      `sudo cp #{temp_path} #{amazon_path}`
+      cmd = "sudo chown `whoami`:`whoami` #{amazon_path} && sudo chmod 0644 #{amazon_path}"
+      `#{cmd}`
+      # remove the temp file
+      `rm -f #{temp_path}`
+      puts "Your amazon config has been saved to #{amazon_path}"
+    end
+  end
+end

data/lib/commands/delete_instances.rb

@@ -0,0 +1,114 @@
+
+module Commands
+  class DeleteInstances
+
+    # holds the options that were passed
+    # you can set any initial defaults here
+    def options
+      @options ||= {
+          :force_delete => false
+      }
+    end
+
+    # required options
+    def required_options
+      @required_options ||= Set.new [
+          :instances,
+          :group
+      ]
+    end
+
+    def register(opts, global_options)
+      opts.banner = "Usage: delete [options]"
+      opts.description = "Delete server instance(s)"
+
+      opts.on('-i', "--instance instance1,instance2,etc", Array, "The instance(s) to delete.") do |v|
+        options[:instances] = v
+      end
+
+      opts.on('-g', "--group deploy_group", "Required - The deploy group we are in.  A deploy group is a set of servers that are required to run the infrastructure for a server.") do |v|
+        options[:group] = v
+      end
+
+      opts.on('-f', "--force", "Force a deletion of a restricted app type (such as db or app_master).  App will most likely fail to operate properly after this.") do |v|
+        options[:force_delete] = v
+      end
+
+      opts.on('-w', "--wait", "Wait until the instance has completely shut down and terminated.  Otherwise, we return immediately after starting the terminate.") do |v|
+        options[:wait] = v
+      end
+
+      opts.on('-p', "--print path", "The directory into which we output the data as a file per host.") do |v|
+        options[:result_path] = v
+      end
+    end
+
+
+    def run(global_options, amazon)
+      ec2 = amazon.ec2
+      elb = amazon.elb
+      utils = ZZSharedLib::Utils.new(amazon)
+
+      group_name = options[:group]
+      deploy_group = amazon.find_deploy_group(group_name)
+      group_config = deploy_group.config
+      amazon_elb = group_config[:amazon_elb]
+
+      inst_ids = options[:instances]
+      if options[:force_delete] == false
+        # check to see if restricted type
+        inst_ids.each do |inst_id|
+          flat = amazon.flat_tags_for_resource(inst_id)
+          role = flat[:role]
+          role = role.to_sym unless role.nil?
+          case role
+            when :app_master, :db
+              raise "You cannot delete an instance with a role of #{role}.  Doing so will cause the server to not operate.  You can force with the --force option."
+          end
+        end
+      end
+
+
+      amazon.flush_tags    # force a refresh of the cached tags
+      all_instances = amazon.find_and_sort_named_instances(group_name)
+
+      instances = all_instances.reject { |inst| inst_ids.include?(inst[:resource_id]) == false }
+
+      # call servers to perform clean shutdown
+      puts "Attempting to perform clean shutdown on remote machines.  Will ignore any errors and continue with shutdown."
+      BuildDeployConfig.do_remote_shutdown(utils, amazon, instances, group_name, deploy_group, options[:result_path]) rescue nil
+
+      # mark state on machines as deleted
+      ec2.create_tags(inst_ids, {:state => 'delete' })
+
+      # now shut them down
+      ec2.terminate_instances([inst_ids])
+      inst_ids.each do |inst_id|
+        puts "Instance #{inst_id} is terminating."
+      end
+      if options[:wait] == true
+        puts "Waiting for instances to terminate."
+        while true do
+          print "."
+          STDOUT.flush
+          sleep(1)
+          instances = ec2.describe_instances(inst_ids)
+          all_terminated = true
+          instances.each do |instance|
+            if instance[:aws_state] != "terminated"
+              all_terminated = false
+              break
+            end
+          end
+          break if all_terminated
+        end
+      end
+
+      # we may want to kick off a redeploy of the chef config and app since the configuration of the
+      # group changed.  Right now we leave this up to the caller since they may want to do it after
+      # deleting more than one instance to avoid multiple redeploys
+      puts
+      puts "Since the configuration has changed you should redeploy the configuration and application using chef_bake and deploy."
+    end
+  end
+end