ey-flex-test 0.3.3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Engine Yard Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+== ey
+
+A gem that provides...

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+
+GEM = "ey-flex-test"
+GEM_VERSION = "0.3.3"
+AUTHOR = "Ezra Zygmuntowicz"
+EMAIL = "awsmdev@engineyard.com"
+HOMEPAGE = "http://engineyard.com/solo"
+SUMMARY = "Command line interface to Engine Yard's cloud"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.bindir       = "bin"
+  s.executables  = %w( ey-slave ey-recipes eybackup ey-snapshots ey-monitor ey-agent)
+  # Uncomment this to add a dependency
+  # s.add_dependency "foo"
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.rdoc Rakefile TODO) + Dir.glob("{lib,spec}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end

data/TODO

@@ -0,0 +1,4 @@
+TODO:
+Fix LICENSE with your name
+Fix Rakefile with your name and contact info
+Add your code to lib/<%= name %>.rb

data/bin/ey-agent

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'json'
+require 'big-brother'
+
+json = JSON.parse(IO.read('/etc/chef/dna.json'))
+
+
+# {'skip':[
+#   'mysqld'
+# ],
+# 'check':[
+#   'ttsrv'
+# ]}
+
+skips = JSON.parse(IO.read('/etc/ey-alerts.json')) rescue {}
+
+puts EY::BigBrother.new(json.merge(skips)).check

data/bin/ey-monitor

@@ -0,0 +1,10 @@
+
+require 'rubygems'
+require 'yaml'
+require 'stonith'
+
+EventMachine.run {
+  opts = YAML::load(IO.read("/etc/.mysql.backups.yml"))
+  opts.merge!(YAML::load(IO.read("/etc/.ey-cloud.yml")))
+  EY::Stonith.new(opts.merge(:heartbeat => 10))
+}

data/bin/ey-recipes

@@ -0,0 +1,133 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'yaml'
+require "optparse"
+require "json"
+require 'ey'
+
+defaults = {:config => '~/.ey-cloud.yml',
+           :command => :get_envs,
+           :type => 'recipes',
+           :keep => 5}
+options = {}
+# Build a parser for the command line arguments
+opts = OptionParser.new do |opts|
+  opts.version = "0.0.1"
+
+  opts.banner = "Usage: ey-recipes [-flag] [argument]"
+  opts.define_head "ey-recipes: managing your recipes..."
+  opts.separator '*'*80
+  
+  opts.on("-l", "--list-recipes ENV", "List recipes for ENV") do |env|
+    options[:env] = env
+    options[:command] = :list
+  end
+  
+  opts.on("-j", "--json ENV", "Get The DNA JSON for ENV") do |env|
+    options[:env] = env
+    options[:command] = :get_json
+  end
+  
+  opts.on("-q", "--quick", "Run only the recipes required for a quick redeploy") do
+    options[:quick] = true
+  end
+  
+  opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+    options[:config] = config
+  end
+  
+  opts.on("-r", "--rollback ENV", "Roll back to the previous recipe set for ENV and run them on your instances.") do |env|
+    options[:env] = env
+    options[:command] = :rollback
+  end
+  
+  opts.on("-d", "--deploy ENV", "Upload a new recipe set and run them on your instances. Be sure to commit your changes to your git repo before deploying as we create a git archive of HEAD.") do |env|
+    options[:command] = :deploy
+    options[:identifier] = 'recipes'
+    options[:env] = env
+  end
+  
+  opts.on("-u", "--upload ENV", "Upload a new recipe set so s3 but don't run it. Be sure to commit your changes to your git repo before deploying as we create a git archive of HEAD.") do |env|
+    options[:command] = :upload
+    options[:identifier] = 'recipes'
+    options[:env] = env
+  end
+  
+  opts.on("--clear ENV", "Clear out any customer recipes attached to ENV") do |env|
+    options[:command] = :clear
+    options[:identifier] = 'recipes'
+    options[:env] = env
+  end
+  
+  opts.on("--deploy-main ENV", "Redeploy the main EY recipe set and run it on your environment ENV") do |env|
+    options[:command] = :deploy_main
+    options[:env] = env
+  end
+  
+  opts.on("-n", "--newest ENV", "download, install and run the current custom recipe set for ENV (THIS COMMAND CAN ONLY BE RUN ON YOUR EC2 INSTANCE)") do |env|
+    options[:command] = :converge
+    options[:identifier] = 'recipes'
+    options[:env] = env
+  end
+  
+  opts.on("--view-log ENV", "view the last custom chef recipe run log file for ENV") do |env|
+    options[:command] = :view_logs
+    options[:env] = env
+  end
+  
+  opts.on("--view-main-log ENV", "view the last main ey recipe run log file for ENV") do |env|
+    options[:command] = :view_logs
+    options[:env] = env
+    options[:main] = true
+  end
+  
+  opts.on("--main ENV", "download, install and run the main ey recipe set for ENV (THIS COMMAND CAN ONLY BE RUN ON YOUR EC2 INSTANCE)") do |env|
+    options[:command] = :converge
+    options[:env] = env
+    options[:main] = true
+    options[:recipeloc] = "/etc/chef/recipes"
+  end
+  
+  opts.on("--logs", "set the type to logs") do
+    options[:type] = 'logs'
+    options[:extension] = 'gz'
+  end
+  
+end
+
+opts.parse!
+
+ey = nil
+if File.exist?(config = File.expand_path(options[:config] || defaults[:config]))
+  ey = EY::ChefRecipes.new(options = defaults.merge(YAML::load(IO.read(config))).merge(options))
+elsif File.exist?(config = "/etc/.ey-cloud.yml")
+  ey = EY::ChefRecipes.new(options = defaults.merge(YAML::load(IO.read(config))).merge(options))
+else
+  puts"You need to have an ~/.ey-cloud.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c path/to/ey-cloud.yml"
+  exit 1
+end
+
+case options[:command]
+when :list
+  ey.list true
+when :rollback
+  ey.rollback
+when :deploy
+  ey.deploy
+when :get_json
+  jj ey.get_json
+when :deploy_main
+  ey.deploy_main
+when :view_logs
+  ey.view_logs  
+when :clear
+  ey.clear_bucket
+when :upload
+  ey.upload_recipes
+when :get_envs
+  envs = ey.get_envs
+  puts "Current Environments:"
+  envs.each {|k,v| puts "env: #{k}  running instances: #{v['instances']}\n  instance_ids: #{v['instance_ids'].inspect}" }
+when :converge
+  ey.converge
+end

data/bin/ey-slave

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'json'
+require 'mysql_slave'
+
+puts "Adding this node as a mysql slave/replicant read only database"
+
+EY::MysqlSlave.new.bootstrap

data/bin/ey-snapshots

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'yaml'
+require "optparse"
+require "json"
+require 'snapshot_minder'
+
+defaults = {:config => '/etc/.mysql.backups.yml',
+           :command => :list_snapshots,
+           :keep => 5}
+
+options = {}
+# Build a parser for the command line arguments
+opts = OptionParser.new do |opts|
+  opts.version = "0.0.1"
+
+  opts.banner = "Usage: ey-snapshots [-flag] [argument]"
+  opts.define_head "ey-snapshots: managing your snapshots..."
+  opts.separator '*'*80
+  
+  opts.on("-l", "--list-snapshots", "list snapshots") do 
+    options[:command] = :list_snapshots
+  end
+
+  opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+    options[:config] = config
+  end
+  
+  opts.on("-i", "--instance-id ID", "specify the instance id to work with(only needed if you are running this from ourside of ec2)") do |iid|
+    options[:instance_id] = iid
+  end
+  
+  
+  opts.on("--snapshot", "take snapshots of both of your volumes(only runs on your ec2 instance)") do
+    options[:command] = :snapshot_volumes
+  end
+
+  
+end
+
+opts.parse!
+
+ey = nil
+if File.exist?(config = File.expand_path(defaults[:config]))
+  ey =  EY::SnapshotMinder.new(options = defaults.merge(YAML::load(IO.read(config))).merge(options))
+else
+  puts"You need to have an /etc/.mysql.backups.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c .mysql.backups.yml"
+  exit 1
+end
+
+ey.send(options[:command])
+ey.clean_snapshots(options[:keep])

data/bin/eybackup

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'yaml'
+require "optparse"
+require 'mysql_backup'
+require 'postgresql_backup'
+
+options = {:config => '/etc/.mysql.backups.yml',
+           :command => :new_backup}
+
+# Build a parser for the command line arguments
+opts = OptionParser.new do |opts|
+  opts.version = "0.0.1"
+
+  opts.banner = "Usage: eybackup [-flag] [argument]"
+  opts.define_head "eybackup: backing up your shit since way back when..."
+  opts.separator '*'*80
+  
+  opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
+    options[:db] = (db || 'all')
+    options[:command] = :list
+  end
+  
+  opts.on("-n", "--new-backup", "Create new mysql backup") do
+    options[:command] = :new_backup
+  end
+  
+  opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+    options[:config] = config
+  end
+  
+  opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index|
+    options[:command] = :download
+    options[:index] = index
+  end
+
+  opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
+    options[:engine] = engine || 'mysql'
+    options[:config] ||= '/etc/.#{options[:engine]}.backups.yml'
+  end
+
+  opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index|
+    options[:command] = :restore
+    options[:index] = index
+  end
+
+end
+
+opts.parse!
+
+eyb = nil
+if File.exist?(options[:config])
+  eyb = case options[:engine]
+    when 'postgres'         then EyBackup::PostgresqlBackup.new(YAML::load(IO.read(options[:config])))
+    when 'mysql', NilClass  then EyBackup::MysqlBackup.new(YAML::load(IO.read(options[:config])))
+    else raise "Invalid engine: #{options[:engine]}"
+    end
+end
+
+case options[:command]
+when :list
+  eyb.list options[:db], true
+when :new_backup
+  eyb.new_backup
+when :download
+  eyb.download(options[:index])
+when :restore
+  eyb.restore(options[:index])
+end
+eyb.cleanup

data/lib/big-brother.rb

@@ -0,0 +1,66 @@
+require 'dbi'
+module EY
+  class BigBrother
+    def initialize(dna)
+      @dna = dna
+      @result = {}
+    end
+    
+    def nginx_or_apache
+      server = ''
+      @dna['applications'].each do |name, app_data|
+        if app_data['recipes'].detect { |r| r == 'nginx' }
+          server = 'nginx'
+        end
+        
+        if app_data['recipes'].detect { |r| r == 'passenger' }
+          server = 'apache2'
+        end
+      end
+      server
+    end
+    
+    def skip?(name)
+      (@dna['skip']||[]).include?(name)
+    end
+    
+    def check
+      case @dna['instance_role']
+      when 'solo'
+        check_process(nginx_or_apache) unless skip?(nginx_or_apache)
+        check_mysql unless skip?('mysqld')
+      when 'app', 'app_master'
+        check_process(nginx_or_apache) unless skip?(nginx_or_apache)
+        check_process('haproxy') unless skip?('haproxy')
+      when 'db_master', 'db_slave'
+        check_mysql unless skip?('mysqld')
+      when 'util'
+      end
+      (@dna['check']||[]).each do |check|
+        check_process(check)
+      end
+      @result.to_json
+    end
+    
+    def check_mysql
+      check_process('mysqld')
+      DBI.connect("DBI:Mysql:mysql:#{@dna['db_host']}", 'root', @dna['users'].first['password'])
+    rescue DBI::DatabaseError => e
+      @result['mysqld'] = 'down'
+    end
+    
+    def check_process(name)
+      return if name == ''
+      pids = `pgrep #{name}`.split("\n")
+      if pids.empty?
+        @result[name] = 'down'
+      else
+        if pids.detect {|p| `kill -0 #{p}; echo $?`.chomp.to_i != 0}
+          @result[name] = 'down'
+        else
+          @result[name] = 'up'
+        end
+      end
+    end
+  end
+end

data/lib/bucket_minder.rb

@@ -0,0 +1,110 @@
+module AWS::S3
+  class S3Object
+    def <=>(other)
+      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
+    end
+  end
+end
+
+require 'open-uri'
+
+module EY
+
+  class BucketMinder
+    
+    def initialize(opts={})
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => opts[:aws_secret_id],
+          :secret_access_key => opts[:aws_secret_key]
+        )
+      @instance_id = opts[:instance_id]
+      @type = opts[:type]
+      @env  = opts[:env]
+      @opts = opts
+      opts[:extension] ||= "tgz"
+      @keep = opts[:keep]
+      @name = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.#{@type}.#{opts[:extension]}"
+    end
+    
+    def bucket
+      @bucket ||= begin
+        buck = "#{@env}-#{@type}-#{instance_id}-#{Digest::SHA1.hexdigest(@opts[:aws_secret_id])[0..6]}"
+          begin
+            AWS::S3::Bucket.create buck
+          rescue AWS::S3::ResponseError
+          end
+        buck
+      end
+    end
+    
+    def instance_id
+      @instance_id ||= open("http://169.254.169.254/latest/meta-data/instance-id").read
+    end
+    
+    def upload_object(file)
+      AWS::S3::S3Object.store(
+         @name,
+         open(file),
+         bucket,
+         :access => :private
+      )
+      FileUtils.rm file
+      puts "successful upload: #{@name}"
+      true
+    end
+    
+    def download(index, printer = false)
+      obj =  list[index.to_i]
+      puts "downloading: #{obj}" if printer
+      File.open(obj.key, 'wb') do |f|
+        print "." if printer
+        obj.value {|chunk| f.write chunk }
+      end
+      puts if printer
+      puts "finished" if printer
+      obj.key
+    end
+    
+    def cleanup
+      list[0...-(@keep)].each{|o| 
+        puts "deleting: #{o.key}"  
+        o.delete
+      }
+    end
+    
+    def get_current
+      name = download(list.size - 1)
+      File.expand_path(name)
+    end
+    
+    def clear_bucket
+      list.each do |o|
+        puts "deleting: #{o.key}" 
+        o.delete
+      end  
+    end
+    
+    def rollback
+      o = list.last
+      puts "rolling back: #{o.key}"  
+      o.delete
+    end
+    
+    def empty?
+      list.empty?
+    end
+    
+    def list(printer = false)
+      objects = AWS::S3::Bucket.objects(bucket).sort
+      puts "listing bucket #{bucket}" if printer && !objects.empty?
+      if printer
+        objects.each_with_index do |b,i|
+          puts "#{i}:#{@env} #{b.key}"
+        end
+      end    
+      objects
+    end
+    
+  end
+  
+end