ey_recipes 0.9.1

data/bin/ey-recipes

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'ey_recipes'
+
+EY::Recipes.run(ARGV)

data/lib/ey_recipes/api.rb

@@ -0,0 +1,56 @@
+module EY::Recipes
+  class API
+    def self.connect(*args)
+      new(*args).connect
+    end
+
+    def initialize(api_url, aws_secret_id, aws_secret_key)
+      @api_url, @aws_secret_id, @aws_secret_key = api_url, aws_secret_id, aws_secret_key
+    end
+
+    def connect
+      @rest = RestClient::Resource.new(@api_url)
+      self
+    end
+
+    def get_envs
+      @envs ||= call_api("environments")
+    end
+
+    def get_json(instance_id)
+      call_api("json_for_instance", :instance_id => instance_id)
+    end
+
+    def deploy_recipes(type, env_id)
+      response = call_api(path_for(type), :id => env_id)
+      response[0] == 'working'
+    end
+
+    private
+
+    def path_for(type)
+      case type
+      when :main:   'deploy_main_recipes'
+      when :custom: 'deploy_recipes'
+      else
+        abort "Unknown recipe type: #{type}"
+      end
+    end
+
+    def keys
+      @keys ||= {:aws_secret_id => @aws_secret_id, :aws_secret_key => @aws_secret_key}
+    end
+
+    def call_api(path, opts={})
+      JSON.parse(@rest["/api/#{path}"].post(opts.merge(keys), {"Accept" => "application/json"}))
+    rescue RestClient::RequestFailed => e
+      case e.http_code
+      when 503
+        sleep 10 # Nanite, save us...
+        retry
+      else
+        raise "API call to Engine Yard failed with status #{e.http_code}."
+      end
+    end
+  end
+end

data/lib/ey_recipes/bucket_minder.rb

@@ -0,0 +1,108 @@
+require 'open-uri'
+
+module EY::Recipes
+  class BucketMinder
+    def initialize(opts={})
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => opts[:aws_secret_id],
+          :secret_access_key => opts[:aws_secret_key]
+        )
+      @instance_id = opts[:instance_id]
+      @type = opts[:type]
+      @env  = opts[:env]
+      @opts = opts
+      opts[:extension] ||= "tgz"
+      @keep = opts[:keep]
+      @name = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.#{@type}.#{opts[:extension]}"
+    end
+    
+    def bucket
+      @bucket ||= begin
+        buck = "#{@env}-#{@type}-#{@instance_id}-#{Digest::SHA1.hexdigest(@opts[:aws_secret_id])[0..6]}"
+          begin
+            AWS::S3::Bucket.create buck
+          rescue AWS::S3::ResponseError
+          end
+        buck
+      end
+    end
+
+    def upload_object(file)
+      AWS::S3::S3Object.store(
+         @name,
+         open(file),
+         bucket,
+         :access => :private
+      )
+      FileUtils.rm file
+      puts "successful upload: #{@name}"
+      true
+    end
+    
+    def download(index, printer = false)
+      obj =  list[index.to_i]
+      puts "downloading: #{obj}" if printer
+      File.open(obj.key, 'wb') do |f|
+        print "." if printer
+        obj.value {|chunk| f.write chunk }
+      end
+      puts if printer
+      puts "finished" if printer
+      obj.key
+    end
+    
+    def cleanup
+      begin
+        list[0...-(@keep)].each do |o| 
+          puts "deleting: #{o.key}"  
+          o.delete
+        end
+        # S3's eventual consistency sometimes causes really weird
+        # failures.
+        # Since cleanup happens every time and will clean up all stale
+        # objects, we can just ignore S3-interaction failures. It'll
+        # work next time.
+      rescue AWS::S3::S3Exception, AWS::S3::Error
+        nil
+      end
+    end
+
+    def clear_bucket
+      list.each do |o|
+        puts "deleting: #{o.key}" 
+        o.delete
+      end  
+    end
+    
+    def rollback
+      o = list.last
+      puts "rolling back: #{o.key}"  
+      o.delete
+    end
+
+    def list(printer = false)
+      objects = AWS::S3::Bucket.objects(bucket)
+      object_last_modified = objects.map do |o|
+        # NB: the last-modified value may be nil.
+        [o, o.about['last-modified']]
+      end.inject({}) do |acc, (obj, obj_last_modified)|
+        acc.merge(obj => obj_last_modified)
+      end
+
+      # sort only the remaining ones
+      objects = objects.find_all do |o|
+        object_last_modified[o]
+      end.sort do |a, b|
+        DateTime.parse(object_last_modified[a]) <=> DateTime.parse(object_last_modified[a])
+      end
+
+      puts "listing bucket #{bucket}" if printer && !objects.empty?
+      if printer
+        objects.each_with_index do |b,i|
+          puts "#{i}:#{@env} #{b.key}"
+        end
+      end
+      objects
+    end
+  end
+end

data/lib/ey_recipes/cli.rb

@@ -0,0 +1,259 @@
+module EY::Recipes
+  class CLI
+    def abort(message)
+      raise message
+    end
+
+    LOG_FILE = "/var/log/chef.log"
+    def self.run(args)
+      $stdout.sync = true
+
+      command = :list_envs
+      defaults = {
+        :config => '~/.ey-cloud.yml',
+        :type => 'recipes',
+        :keep => 5
+      }
+
+      options = {}
+      # Build a parser for the command line arguments
+      opts = OptionParser.new do |opts|
+        opts.version = "0.0.1"
+
+        opts.banner = "Usage: ey-recipes [-flag] [argument]"
+        opts.define_head "ey-recipes: managing your recipes..."
+        opts.separator '*'*80
+
+        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+          options[:config] = config
+        end
+
+        opts.on("-l", "--list-recipes ENV", "List recipes for ENV") do |env|
+          command = :list_recipes
+          options[:env_name] = env
+        end
+
+        opts.on("-u", "--upload ENV", "Upload a new recipe set so s3 but don't run it. Be sure to commit your changes to your git repo before deploying as we create a git archive of HEAD.") do |env|
+          command = :upload
+          options[:env_name] = env
+        end
+
+        opts.on("--clear ENV", "Clear out any customer recipes attached to ENV") do |env|
+          command = :clear
+          options[:env_name] = env
+        end
+
+        opts.on("-d", "--deploy ENV", "Upload a new recipe set and run them on your instances. Be sure to commit your changes to your git repo before deploying as we create a git archive of HEAD.") do |env|
+          command = :deploy_custom
+          options[:env_name] = env
+        end
+
+        opts.on("-r", "--rollback ENV", "Roll back to the previous recipe set for ENV and run them on your instances.") do |env|
+          command = :rollback
+          options[:env_name] = env
+        end
+
+        opts.on("--deploy-main ENV", "Redeploy the main EY recipe set and run it on your environment ENV") do |env|
+          command = :deploy_main
+          options[:env_name] = env
+        end
+
+        opts.on("--view-main-log ENV", "view the last main ey recipe run log file for ENV") do |env|
+          command = :view_main_logs
+          options[:env_name] = env
+        end
+
+        opts.on("--view-log ENV", "view the last custom chef recipe run log file for ENV") do |env|
+          command = :view_custom_logs
+          options[:env_name] = env
+        end
+      end
+
+      begin
+        opts.parse!(args)
+      rescue OptionParser::ParseError => e
+        abort(e.message)
+      end
+
+      ey = nil
+      if File.exist?(config = File.expand_path(options[:config] || defaults[:config]))
+        ey = new(options = defaults.merge(YAML::load(File.read(config))).merge(options))
+      else
+        puts "You need to have an ~/.ey-cloud.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c path/to/ey-cloud.yml"
+        exit 1
+      end
+
+      if ey.respond_to?(command)
+        ey.send(command)
+      else
+        abort "Unknown command: #{command}"
+      end
+    end
+
+    def initialize(opts)
+      @opts = opts
+      @api = API.connect(opts[:api], opts[:aws_secret_id], opts[:aws_secret_key])
+      @lock = Mutex.new
+    end
+
+    def list_recipes
+      recipes_bucket.list(true)
+    end
+
+    def list_envs
+      puts "Current Environments:"
+      envs.each do |name,data|
+        puts "env: #{name}  running instances: #{data['instances']}"
+        puts "  instance_ids: #{data['instance_ids'].inspect}"
+      end
+    end
+
+    def upload
+      upload_recipes
+    end
+
+    def clear
+      recipes_bucket.clear_bucket
+    end
+
+    def deploy_custom
+      upload_recipes
+      deploy_custom_recipes
+    end
+
+    def deploy_custom_recipes
+      deploy_recipes(:custom, 'logs')
+    end
+
+    def deploy_main
+      deploy_recipes(:main, 'main.logs')
+    end
+
+    def rollback
+      recipes_bucket.rollback
+      deploy_custom_recipes
+    end
+
+    def view_main_logs
+      view_logs('main.logs')
+    end
+
+    def view_custom_logs
+      view_logs('logs')
+    end
+
+    def get_json
+      @api.get_json(@opts[:instance])
+    end
+
+    def deploy_recipes(type, log_type)
+      unless env['instances'] > 0
+        abort "There are no running instances for ENV: #{env_name}"
+      end
+
+      puts "deploying #{type} recipes..."
+      if @api.deploy_recipes(type, env['id'])
+        wait_for_logs(log_type)
+      else
+        abort "deploying #{type} recipes failed..."
+      end
+    end
+
+    def upload_recipes
+      unless File.exist?("cookbooks")
+        abort "you must run this command from the root of your chef recipe git repo"
+      end
+
+      file = "recipes.#{$$}.#{Time.now.to_i}.tmp.tgz"
+      tarcmd = "git archive --format=tar HEAD | gzip > #{file}"
+      if system(tarcmd)
+        recipes_bucket.upload_object(file)
+        recipes_bucket.cleanup
+      else
+        abort "Failed to upload recipes for #{env_name}"
+      end
+    end
+
+    def view_logs(log_type)
+      env['instance_ids'].each do |instance_id|
+        log_bucket = log_bucket_for(log_type, instance_id)
+        puts "Logs for: #{instance_id}"
+        puts display_logs(log_bucket.list.last)
+      end
+    end
+
+    private
+
+    def recipes_bucket
+      @recipes_bucket ||= bucket_for('recipes', 'recipes', 'tgz')
+    end
+
+    def envs
+      @envs ||= @api.get_envs
+    end
+
+    def env_name
+      @opts[:env_name] || abort("must provide environment name")
+    end
+
+    def env
+      @env ||= find_env(env_name)
+    end
+
+    def find_env(env_name)
+      unless env = envs[env_name]
+        abort "#{env} is not a valid environment name, your available environments are:\n#{envs.keys.join("\n")}"
+      end
+      env
+    end
+
+    def wait_for_logs(log_type)
+      threads = []
+      env['instance_ids'].each do |instance_id|
+        threads << Thread.new { wait_for_log(log_type, instance_id) }
+      end
+      threads.each {|t| t.join}
+    end
+
+    def wait_for_log(log_type, instance_id)
+      logbucket = log_bucket_for(log_type, instance_id)
+      newest = @lock.synchronize { logbucket.list.last }
+      count = 0
+      until @lock.synchronize { newest != logbucket.list.last }
+        print "."
+        sleep 3
+        count += 1
+        if count > 600
+          puts "timed out waiting for deployed logs"
+          exit 1
+        end  
+      end
+      puts
+      puts "retrieving logs..."
+      puts @lock.synchronize { display_logs(logbucket.list.last) }
+    end
+
+    def log_bucket_for(log_type, instance_id)
+      bucket_for(log_type, instance_id, 'gz')
+    end
+
+    def bucket_for(log_type, instance_id, extension)
+      BucketMinder.new(:aws_secret_key  => @opts[:aws_secret_key],
+                       :aws_secret_id   => @opts[:aws_secret_id],
+                       :keep            => @opts[:keep],
+                       :env             => env_name,
+                       :type            => log_type,
+                       :extension       => extension,
+                       :instance_id     => instance_id)
+    end
+
+    def display_logs(obj)
+      if obj
+        Zlib::GzipReader.new(StringIO.new(obj.value, 'rb')).read
+      else
+        "no logs..."
+      end
+    end
+
+  end
+end

data/lib/ey_recipes/version.rb

@@ -0,0 +1,3 @@
+module EY::Recipes
+  VERSION = "0.9.1"
+end

data/lib/ey_recipes.rb

@@ -0,0 +1,18 @@
+require 'json'
+require 'restclient'
+require 'aws/s3'
+require "optparse"
+
+module EY
+  module Recipes
+    def self.run(args)
+      CLI.run(args)
+    end
+  end
+end
+
+current_dir = File.expand_path(File.dirname(__FILE__) + '/ey_recipes')
+require current_dir + '/bucket_minder'
+require current_dir + '/api'
+require current_dir + '/cli'
+require current_dir + '/version'

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: ey_recipes
+version: !ruby/object:Gem::Version 
+  version: 0.9.1
+platform: ruby
+authors: 
+- Ninjas
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-20 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Gem for kicking off chef recipes
+email: awsmdev@engineyard.com
+executables: 
+- ey-recipes
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/ey_recipes/api.rb
+- lib/ey_recipes/bucket_minder.rb
+- lib/ey_recipes/cli.rb
+- lib/ey_recipes/version.rb
+- lib/ey_recipes.rb
+has_rdoc: true
+homepage: http://github.com/engineyard/ey_recipes
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Gem for kicking off chef recipes
+test_files: []
+