ey-beta 0.0.4

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Engine Yard Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+== ey
+
+A gem that provides...

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+
+GEM = "ey-beta"
+GEM_VERSION = "0.0.4"
+AUTHOR = "Ezra Zygmuntowicz"
+EMAIL = "ez@engineyard.com"
+HOMEPAGE = "http://engineyard.com/solo"
+SUMMARY = "Command line interface to Engine Yard's cloud"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.bindir       = "bin"
+  s.executables  = %w( ey-recipes  )
+  # Uncomment this to add a dependency
+  # s.add_dependency "foo"
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,spec}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end

data/TODO

@@ -0,0 +1,4 @@
+TODO:
+Fix LICENSE with your name
+Fix Rakefile with your name and contact info
+Add your code to lib/<%= name %>.rb

data/bin/ey-recipes

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'yaml'
+require "optparse"
+require "json"
+require 'ey'
+
+defaults = {:config => '~/.ey-cloud.yml',
+           :command => :get_envs,
+           :type => 'recipes',
+           :keep => 5}
+options = {}
+# Build a parser for the command line arguments
+opts = OptionParser.new do |opts|
+  opts.version = "0.0.1"
+
+  opts.banner = "Usage: ey-recipes [-flag] [argument]"
+  opts.define_head "ey-recipes: managing your recipes..."
+  opts.separator '*'*80
+  
+  opts.on("-l", "--list-recipes ENV", "List recipes for ENV") do |env|
+    options[:env] = env
+    options[:command] = :list
+  end
+  
+  opts.on("-j", "--json ENV", "Get The DNA JSON for ENV") do |env|
+    options[:env] = env
+    options[:command] = :get_json
+  end
+  
+  opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+    options[:config] = config
+  end
+  
+  opts.on("-r", "--rollback ENV", "Roll back to the previous recipe set for ENV and run them on your instances.") do |env|
+    options[:env] = env
+    options[:command] = :rollback
+  end
+  
+  opts.on("-d", "--deploy ENV", "Upload a new recipe set and run them on your instances. Be sure to commit your changes to your git repo before deploying as we create a git archive of HEAD.") do |env|
+    options[:command] = :deploy
+    options[:env] = env
+  end
+  
+  opts.on("--deploy-main ENV", "Redeploy the main EY recipe set and run it on your environment ENV") do |env|
+    options[:command] = :deploy_main
+    options[:env] = env
+  end
+  
+  opts.on("-n", "--newest ENV", "download, install and run the current custom recipe set for ENV (THIS COMMAND CAN ONLY BE RUN ON YOUR EC2 INSTANCE)") do |env|
+    options[:command] = :converge
+    options[:env] = env
+  end
+  
+  opts.on("--view-log ENV", "view the last custom chef recipe run log file for ENV") do |env|
+    options[:command] = :view_logs
+    options[:env] = env
+  end
+  
+  opts.on("--view-main-log ENV", "view the last main ey recipe run log file for ENV") do |env|
+    options[:command] = :view_logs
+    options[:env] = env
+    options[:main] = true
+  end
+  
+  opts.on("--main ENV", "download, install and run the main ey recipe set for ENV (THIS COMMAND CAN ONLY BE RUN ON YOUR EC2 INSTANCE)") do |env|
+    options[:command] = :converge
+    options[:env] = env
+    options[:main] = true
+    options[:recipeloc] = "/etc/chef/recipes"
+  end
+  
+  opts.on("--logs", "set the type to logs") do
+    options[:type] = 'logs'
+    options[:extension] = 'gz'
+  end
+  
+end
+
+opts.parse!
+
+ey = nil
+if File.exist?(config = File.expand_path(defaults[:config]))
+  ey = EY::ChefRecipes.new(options = defaults.merge(YAML::load(IO.read(config))).merge(options))
+elsif File.exist?(config = "/etc/.ey-cloud.yml")
+  ey = EY::ChefRecipes.new(options = defaults.merge(YAML::load(IO.read(config))).merge(options))
+else
+  puts"You need to have an ~/.ey-cloud.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c path/to/ey-cloud.yml"
+  exit 1
+end
+
+case options[:command]
+when :list
+  ey.list true
+when :rollback
+  ey.rollback
+when :deploy
+  ey.deploy
+when :get_json
+  jj ey.get_json
+when :deploy_main
+  ey.deploy_main
+when :view_logs
+  ey.view_logs  
+when :get_envs
+  envs = ey.get_envs
+  puts "Current Environments:"
+  envs.each {|k,v| puts "env: #{k}  running instances: #{v['instances']}" }
+when :converge
+  ey.converge
+end

data/lib/bucket_minder.rb

@@ -0,0 +1,92 @@
+module AWS::S3
+  class S3Object
+    def <=>(other)
+      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
+    end
+  end
+end
+
+module EY
+
+  class BucketMinder
+    
+    def initialize(opts={})
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => opts[:aws_secret_id],
+          :secret_access_key => opts[:aws_secret_key]
+        )
+      @type = opts[:type]
+      @env  = opts[:env]
+      opts[:extension] ||= "tgz"
+      @keep = opts[:keep]
+      @bucket = "#{@env}-#{@type}-#{Digest::SHA1.hexdigest(opts[:aws_secret_id])[0..6]}"
+      @name = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.#{@type}.#{opts[:extension]}"
+      begin
+        AWS::S3::Bucket.create @bucket
+      rescue AWS::S3::ResponseError
+      end  
+    end
+    
+    def upload_object(file)
+      AWS::S3::S3Object.store(
+         @name,
+         open(file),
+         @bucket,
+         :access => :private
+      )
+      FileUtils.rm file
+      puts "successful upload: #{@name}"
+      true
+    end
+    
+    def download(index, printer = false)
+      obj =  list[index.to_i]
+      puts "downloading: #{obj}" if printer
+      File.open(obj.key, 'wb') do |f|
+        print "." if printer
+        obj.value {|chunk| f.write chunk }
+      end
+      puts if printer
+      puts "finished" if printer
+      obj.key
+    end
+    
+    def cleanup
+      list[0...-(@keep)].each{|o| 
+        puts "deleting: #{o.key}"  
+        o.delete
+      }
+    end
+    
+    def get_current
+      name = download(list.size - 1)
+      File.expand_path(name)
+    end
+    
+    def clear_bucket
+      list.each do |o|
+        puts "deleting: #{o.key}" 
+        o.delete
+      end  
+    end
+    
+    def rollback
+      o = list.last
+      puts "rolling back: #{o.key}"  
+      o.delete
+    end
+    
+    def list(printer = false)
+      objects = AWS::S3::Bucket.objects(@bucket).sort
+      puts "listing bucket #{@bucket}" if printer && !objects.empty?
+      if printer
+        objects.each_with_index do |b,i|
+          puts "#{i}:#{@env} #{b.key}"
+        end
+      end    
+      objects
+    end
+    
+  end
+  
+end    

data/lib/ey.rb

@@ -0,0 +1,273 @@
+require 'rubygems'
+require 'aws/s3'
+require 'date'
+require 'digest'
+require 'fileutils'
+require File.join(File.dirname(__FILE__), 'bucket_minder')
+require 'rest_client'
+require 'json'
+require 'open-uri'
+require 'zlib'
+require 'stringio'
+
+$stdout.sync = true
+
+module EY
+
+  class ChefRecipes
+    def initialize(opts={})
+      raise ArgumentError.new("must provide environment name") unless opts[:env] or opts[:command] == :get_envs
+      @opts = opts
+      @eyenv = opts[:eyenv] || 'production'
+      @env = opts[:env]
+      @recipeloc = opts[:recipeloc] || "/etc/chef-custom/recipes"
+      @rest = RestClient::Resource.new(opts[:api])
+      @keys = {:aws_secret_id => @opts[:aws_secret_id], :aws_secret_key => @opts[:aws_secret_key]}
+      @bucket = BucketMinder.new(@opts)
+      unless get_envs[@env] or opts[:command] == :get_envs
+        puts %Q{#{@env} is not a valid environment name, your available environments are:\n#{get_envs.keys.join("\n")}}
+        exit 1
+      end  
+    end
+    
+    def call_api(path, opts={})
+      JSON.parse(@rest["/api/#{path}"].post(@keys.merge(opts)))
+    rescue RestClient::RequestFailed
+      puts "API call to Engine Yard failed. Are there any running instances for #{@env}"
+      exit 1
+    end
+    
+    def get_envs
+      @_envs ||= call_api("environments")
+    end
+    
+    def get_json(instance_id = nil)
+      env = get_envs[@env]
+      call_api("json_for_instance", :id => env['id'], :instance_id => instance_id)
+    rescue
+      {}  
+    end
+    
+    def converge
+      require 'chef'
+      require 'chef/client'
+      FileUtils.mkdir_p @recipeloc
+      logtype = nil
+      build_problem = false
+      out = log_to_string do
+        begin
+          instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").gets
+          defaults = {
+            :log_level => :info,
+            :solo => true,
+            :cookbook_path => "#{@recipeloc}/cookbooks",
+            :file_store_path => "#{@recipeloc}/",
+            :file_cache_path => "#{@recipeloc}/",
+            :node_name => instance_id
+          }
+          Chef::Config.configure { |c| c.merge!(defaults) }
+          Chef::Log::Formatter.show_time = false
+          Chef::Log.level(Chef::Config[:log_level])
+          
+          if File.exist?("#{@recipeloc}/cookbooks")
+            FileUtils.rm_rf("#{@recipeloc}/cookbooks")
+          end
+          
+          if @opts[:main]
+            logtype = "main.logs"
+            install_main_recipes
+          else
+            logtype = "logs"
+            install_recipes
+          end
+          json = get_json(instance_id)
+          File.open("/etc/chef/dna.json", 'w'){|f| f.puts JSON.pretty_generate(json)}
+          c = Chef::Client.new
+          c.json_attribs = json
+          c.run_solo
+        rescue Object => e
+          build_problem = true
+          Chef::Log.error(describe_error(e))
+        end  
+      end
+      file = "/tmp/chef-#{rand(1000)}.log"
+      File.open(file, 'w'){ |f| f.write out }
+      
+      @bucket = BucketMinder.new(@opts.merge(:type => logtype, :extension => 'gz'))
+      upload_logs(file)
+      @bucket.cleanup
+      FileUtils.rm(file)
+      exit 1 if build_problem
+    end
+    
+    def install_main_recipes
+      unless @opts[:main_recipes]
+        puts "you must specify :main_recipes: in your ey-cloud.yml"
+        exit 1
+      end  
+      recipes_path = Chef::Config[:cookbook_path].gsub(/cookbooks/, '')
+      FileUtils.mkdir_p recipes_path
+      path = File.join(recipes_path, 'recipes.tgz')
+      File.open(path, 'wb') do |f|
+        f.write open(@opts[:main_recipes]).read
+      end
+      system("cd #{recipes_path} && tar xzf #{path}")
+      FileUtils.rm path
+    end
+    
+    def deploy
+      unless File.exist?("cookbooks")
+        puts "you must run this command from the root of your chef recipe git repo"
+        exit 1
+      end
+      env = get_envs[@env]
+      unless env['instances'] > 0
+        puts "There are no running instances for ENV: #{@env}"
+        exit 1
+      end
+      if upload_recipes
+        if env
+          puts "deploying recipes..."
+          if call_api("deploy_recipes", :id => env['id'] )[0] == 'working'
+            wait_for_logs('logs')
+          else
+            puts "deploying main recipes failed..."
+          end
+        else
+          puts "No matching environments"
+        end  
+      else
+        puts "Failed to deploy: #{@env}"
+      end  
+    end
+    
+    def rollback
+      @bucket.rollback
+      env = get_envs[@env]
+      if env
+        puts "rolling back recipes..."
+        call_api("deploy_recipes", :id => env['id'] )
+        wait_for_logs('logs')
+      else
+        puts "No matching environments for #{@env}"
+      end
+    end
+    
+    def wait_for_logs(logtype)
+      logbucket = BucketMinder.new(@opts.merge(:type => logtype, :extension => 'gz'))
+      newest = logbucket.list.last
+      count = 0
+      until newest != logbucket.list.last
+        print "."
+        sleep 3
+        count += 1
+        if count > 600
+          puts "timed out waiting for deployed logs"
+          exit 1
+        end  
+      end
+      puts
+      puts "retrieving logs..."
+      puts display_logs(logbucket.list.last)
+    end
+    
+    def deploy_main
+      env = get_envs[@env]
+      if env
+        unless env['instances'] > 0
+          puts "There are no running instances for ENV: #{@env}"
+          exit 1
+        end
+        puts "deploying main EY recipes..."
+        if call_api("deploy_main_recipes", :id => env['id'] )[0] == 'working'
+          wait_for_logs('main.logs')
+        else
+          puts "deploying main recipes failed..."
+        end  
+      else
+        puts "No matching environments"
+      end  
+    end
+    
+    def display_logs(obj)
+      if obj
+        Zlib::GzipReader.new(StringIO.new(obj.value, 'rb')).read
+      else
+        "no logs..."
+      end
+    end
+    
+    def view_logs
+      logtype = "#{@opts[:main] ? 'main.' : ''}logs"
+      logbucket = BucketMinder.new(@opts.merge(:type => logtype, :extension => 'gz'))
+      puts display_logs(logbucket.list.last)
+    end
+    
+    def upload_recipes
+      file = "recipes.#{rand(1000)}.tmp.tgz"
+      tarcmd = "git archive --format=tar HEAD | gzip > #{file}"
+      if system(tarcmd)
+        @bucket.upload_object(file)
+        @bucket.cleanup
+        true
+      else
+        puts "Unable to tar up recipes for #{@opts[:env]} wtf?"
+        false
+      end
+    end
+    
+    def upload_logs(file)
+      name = "#{file}.#{rand(1000)}.tgz"
+      tarcmd = "cat #{file} | gzip > #{name}"
+      if system(tarcmd)
+        @bucket.upload_object(name)
+        @bucket.cleanup
+        true
+      else
+        puts "Unable to tar up log files for #{@opts[:env]} wtf?"
+        false
+      end    
+    end
+    
+    def cleanup
+      @bucket.cleanup
+    end
+    
+    def list(*args)
+      @bucket.list(*args)
+    end
+    
+    def download(*args)
+      @bucket.download(*args)
+    end
+    
+    def install_recipes
+      file = get_current
+      Dir.chdir(@recipeloc) {
+        system("tar xzf #{file}")
+      }
+      FileUtils.rm file
+    end
+    
+    def get_current
+      @bucket.get_current
+    end
+    
+    def clear_bucket
+      @bucket.clear_bucket
+    end
+    
+    def describe_error(e)
+      "#{e.class.name}: #{e.message}\n #{e.backtrace.join("\n  ")}"
+    end
+    
+    def log_to_string(&block)
+      output = StringIO.new
+      Chef::Log.init(output)
+      block.call
+      output.string
+    end
+    
+  end
+  
+end

data/lib/snapshot_minder.rb

@@ -0,0 +1,123 @@
+require 'right_aws'
+require 'dbi'
+require 'open-uri'
+module EY
+  class SnapshotMinder
+    def initialize(opts={})
+      @opts = opts
+      @instance_id = opts[:instance_id]
+      @db = Mysql.new('root', opts[:dbpass])
+      @ec2 = RightAws::Ec2.new(opts[:aws_secret_id], opts[:aws_secret_key])
+      get_instance_id
+      find_volume_ids
+    end
+    
+    def find_volume_ids
+      @volume_ids = {}
+      @ec2.describe_volumes.each do |volume|
+        if volume[:aws_instance_id] == @instance_id
+          if volume[:aws_device] == "/dev/sdz1"
+            @volume_ids[:data] = volume[:aws_id]
+          elsif volume[:aws_device] == "/dev/sdz2"
+            @volume_ids[:db] = volume[:aws_id]
+          end    
+        end
+      end
+      puts("Volume IDs are #{@volume_ids.inspect}")
+      @volume_ids
+    end
+    
+    def list_snapshots
+      @snapshot_ids = {}
+      @ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+        @volume_ids.each do |mnt, vol|
+          if snapshot[:aws_volume_id] == vol
+            (@snapshot_ids[mnt] ||= []) << snapshot[:aws_id]
+          end  
+        end
+      end
+      puts("Snapshots #{@snapshot_ids.inspect}")
+      @snapshot_ids
+    end
+    
+    def clean_snapshots(keep=5)
+      list_snapshots
+      @snapshot_ids.each do |mnt, ids|
+        snaps = []
+        @ec2.describe_snapshots(ids).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+          snaps << snapshot
+        end
+        snaps[keep..-1].each do |snapshot|
+          puts "deleting snapshot of /#{mnt}: #{snapshot[:aws_id]}"
+          @ec2.delete_snapshot(snapshot[:aws_id])
+        end
+      end
+      list_snapshots
+    end
+    
+    def snapshot_volumes
+      snaps = []
+      @volume_ids.each do |vol, vid|
+        case vol
+        when :data
+          snaps << create_snapshot(vid)
+        when :db
+          @db.flush_tables_with_read_lock
+          snaps << create_snapshot(vid)
+          @db.unlock_tables
+        end
+      end
+      snaps
+    end
+    
+    def get_instance_id
+      return @instance_id if @instance_id
+      
+      open('http://169.254.169.254/latest/meta-data/instance-id') do |f|
+        @instance_id = f.gets
+      end
+      raise "Cannot find instance id!" unless @instance_id
+      puts("Instance ID is #{@instance_id}")
+      @instance_id
+    end
+    
+    
+    def create_snapshot(volume_id)
+      snap = @ec2.create_snapshot(volume_id)
+      puts("Created snapshot of #{volume_id} as #{snap[:aws_id]}")
+      snap
+    end
+    
+  end
+  
+  class Mysql
+    
+    attr_accessor :dbh
+    
+    def initialize(username, password)
+      @username = username
+      @password = password
+      puts("Connecting to MySQL")
+      @dbh = DBI.connect("DBI:Mysql:mysql", username, password)
+    end
+    
+    def flush_tables_with_read_lock
+      puts("Flushing tables with read lock")
+      @dbh.do("flush tables with read lock")
+      true
+    end
+    
+    def unlock_tables
+      puts("Unlocking tables")
+      @dbh.do("unlock tables")
+      true
+    end
+    
+    def disconnect
+      puts("Disconnecting from MySQL")
+      @dbh.disconnect
+    end
+    
+  end
+  
+end 

data/spec/ey_spec.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "ey" do
+  it "should do nothing" do
+    true.should == true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: ey-beta
+version: !ruby/object:Gem::Version 
+  version: 0.0.4
+platform: ruby
+authors: 
+- Ezra Zygmuntowicz
+autorequire: ey-beta
+bindir: bin
+cert_chain: []
+
+date: 2009-03-04 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: Command line interface to Engine Yard's cloud
+email: ez@engineyard.com
+executables: 
+- ey-recipes
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+- TODO
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
+- lib/bucket_minder.rb
+- lib/ey.rb
+- lib/snapshot_minder.rb
+- spec/ey_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://engineyard.com/solo
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Command line interface to Engine Yard's cloud
+test_files: []
+