ey_cloud_server 1.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Engine Yard Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,25 @@
+== ey
+
+= Development
+
+You need to have gem bundler install
+
+    gem install bundler
+
+Then bundle everything up: 
+
+    gem bundle -- --with-mysql_config
+
+To run the specs, just jump into a "bundled" shell: 
+
+    gem exec bash
+
+Copy the example file to spec/config.yml:
+
+Then you can run specs: 
+
+    spec -c spec
+
+Running the bin files are easy too: 
+
+    bin/ey-recipes

data/bin/ey-agent

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/ey-flex'
+
+puts EY::BigBrother.check

data/bin/ey-monitor

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/ey-flex'
+
+EY::Stonith.run

data/bin/ey-snapshots

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/ey-flex'
+
+puts EY::SnapshotMinder.run(ARGV)

data/bin/eybackup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/ey-flex'
+
+EY::Flex::Backups.run(ARGV)

data/lib/ey-flex/backups.rb

@@ -0,0 +1,226 @@
+module AWS::S3
+  class S3Object
+    def <=>(other)
+      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
+    end
+  end
+end
+
+module EY::Flex
+  class DatabaseEngine
+    def self.register_as(name)
+      EY::Flex::Backups::ENGINES[name] = self
+    end
+
+    def initialize(backups)
+      @backups = backups
+    end
+
+    def dump_database(name)
+      raise "Implement #dump_database in #{self.class}"
+    end
+
+    def dbuser
+      @backups.config[:dbuser]
+    end
+
+    def dbpass
+      @backups.config[:dbpass]
+    end
+  end
+
+  class Backups
+    class BackupNotFound < EY::Flex::Error; end
+
+    def self.run(args)
+      options = {:command => :new_backup}
+
+      # Build a parser for the command line arguments
+      opts = OptionParser.new do |opts|
+        opts.version = "0.0.1"
+
+        opts.banner = "Usage: eybackup [-flag] [argument]"
+        opts.define_head "eybackup: backing up your shit since way back when..."
+        opts.separator '*'*80
+        
+        opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
+          options[:db] = (db || 'all')
+          options[:command] = :list
+        end
+        
+        opts.on("-n", "--new-backup", "Create new mysql backup") do
+          options[:command] = :new_backup
+        end
+        
+        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+          options[:config] = config
+        end
+        
+        opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index|
+          options[:command] = :download
+          options[:index] = index
+        end
+
+        opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
+          options[:engine] = engine
+        end
+
+        opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index|
+          options[:command] = :restore
+          options[:index] = index
+        end
+
+      end
+
+      opts.parse!(args)
+
+      options[:engine] ||= 'mysql'
+      options[:config] ||= "/etc/.#{options[:engine]}.backups.yml"
+
+      eyb = new(options)
+
+      case options[:command]
+      when :list
+        eyb.list options[:db], true
+      when :new_backup
+        eyb.new_backup
+      when :download
+        eyb.download(options[:index])
+      when :restore
+        eyb.restore(options[:index])
+      end
+      eyb.cleanup
+    rescue EY::Flex::Error => e
+      $stderr.puts e.message
+      exit 1
+    end
+
+    ENGINES = {}
+
+    def initialize(options = {})
+      engine_klass = ENGINES[options[:engine]] || raise("Invalid database engine: #{options[:engine].inspect}")
+      @engine = engine_klass.new(self)
+
+      load_config(options[:config])
+
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => config[:aws_secret_id],
+          :secret_access_key => config[:aws_secret_key]
+        )
+      @databases = config[:databases]
+      @keep = config[:keep]
+      @bucket = "ey-backup-#{Digest::SHA1.hexdigest(config[:aws_secret_id])[0..11]}"
+      @tmpname = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.sql.gz"
+      @env = config[:env]
+      FileUtils.mkdir_p '/mnt/backups'
+      FileUtils.mkdir_p '/mnt/tmp'
+      begin
+        AWS::S3::Bucket.find(@bucket)
+      rescue AWS::S3::NoSuchBucket
+        AWS::S3::Bucket.create(@bucket)
+      end
+      
+      FileUtils.mkdir_p self.backup_dir
+    end
+    attr_reader :config
+
+    def load_config(filename)
+      if File.exist?(filename)
+        @config = YAML::load(File.read(filename))
+      else
+        $stderr.puts "You need to have a backup file at #{filename}"
+        exit 1
+      end
+    end
+
+    def new_backup
+      @databases.each do |db|
+        backup_database(db)
+      end  
+    end
+    
+    def backup_database(database)
+      File.open("#{self.backup_dir}/#{database}.#{@tmpname}", "w") do |f|
+        puts "doing database: #{database}"
+        @engine.dump_database(database, f)
+      end
+
+      File.open("#{self.backup_dir}/#{database}.#{@tmpname}") do |f|
+        path = "#{@env}.#{database}/#{database}.#{@tmpname}"
+        AWS::S3::S3Object.store(path, f, @bucket, :access => :private)
+        puts "successful backup: #{database}.#{@tmpname}"
+      end
+    end
+
+    def download(index)
+      idx, db = index.split(":")
+      raise Error, "You didn't specify a database name: e.g. 1:rails_production" unless db
+
+      if obj = list(db)[idx.to_i]
+        filename = normalize_name(obj)
+        puts "downloading: #{filename}"
+        File.open(filename, 'wb') do |f|
+          print "."
+          obj.value {|chunk| f.write chunk }
+        end
+        puts
+        puts "finished"
+        [db, filename]
+      else
+        raise BackupNotFound, "No backup found for database #{db.inspect}: requested index: #{idx}"
+      end
+    end
+    
+    def restore(index)
+      db, filename = download(index)
+      File.open(filename) do |f|
+        @engine.restore_database(db, f)
+      end
+    end
+    
+    def cleanup
+      begin
+        list('all',false)[0...-(@keep*@databases.size)].each do |o| 
+          puts "deleting: #{o.key}"
+          o.delete
+        end
+      rescue AWS::S3::S3Exception, AWS::S3::Error
+        nil # see bucket_minder cleanup note regarding S3 consistency
+      end
+    end
+    
+    def normalize_name(obj)
+      obj.key.gsub(/^.*?\//, '')
+    end
+    
+    def find_obj(name)
+      AWS::S3::S3Object.find name, @bucket
+    end
+    
+    def list(database='all', printer = false)
+      puts "Listing database backups for #{database}" if printer
+      backups = []
+      if database == 'all'
+        @databases.each do |db|
+          backups << AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{db}")
+        end
+        backups = backups.flatten.sort
+      else
+        backups = AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{database}").sort
+      end
+      if printer
+        puts "#{backups.size} backup(s) found"
+        backups.each_with_index do |b,i|
+          puts "#{i}:#{database} #{normalize_name(b)}"
+        end
+      end    
+      backups
+    end
+    
+    protected
+    def backup_dir
+      "/mnt/tmp"
+    end
+    
+  end
+end

data/lib/ey-flex/big-brother.rb

@@ -0,0 +1,80 @@
+module EY
+  class BigBrother
+    def self.check
+      json = JSON.parse(File.read('/etc/chef/dna.json'))
+
+      # {'skip':[
+      #   'mysqld'
+      # ],
+      # 'check':[
+      #   'ttsrv'
+      # ]}
+
+      skips = JSON.parse(File.read('/etc/ey-alerts.json')) rescue {}
+
+      new(json.merge(skips)).check
+    end
+
+    def initialize(dna)
+      @dna = dna
+      @result = {}
+    end
+    
+    def nginx_or_apache
+      server = ''
+      @dna['applications'].each do |name, app_data|
+        if app_data['recipes'].detect { |r| r == 'nginx' }
+          server = 'nginx'
+        end
+        
+        if app_data['recipes'].detect { |r| r == 'passenger' }
+          server = 'apache2'
+        end
+      end
+      server
+    end
+    
+    def skip?(name)
+      (@dna['skip']||[]).include?(name)
+    end
+    
+    def check
+      case @dna['instance_role']
+      when 'solo'
+        check_process(nginx_or_apache) unless skip?(nginx_or_apache)
+        check_mysql unless skip?('mysqld')
+      when 'app', 'app_master'
+        check_process(nginx_or_apache) unless skip?(nginx_or_apache)
+        check_process('haproxy') unless skip?('haproxy')
+      when 'db_master', 'db_slave'
+        check_mysql unless skip?('mysqld')
+      when 'util'
+      end
+      (@dna['check']||[]).each do |check|
+        check_process(check)
+      end
+      @result.to_json
+    end
+    
+    def check_mysql
+      check_process('mysqld')
+      DBI.connect("DBI:Mysql:mysql:#{@dna['db_host']}", 'root', @dna['users'].first['password'])
+    rescue DBI::DatabaseError => e
+      @result['mysqld'] = 'down'
+    end
+    
+    def check_process(name)
+      return if name == ''
+      pids = `pgrep #{name}`.split("\n")
+      if pids.empty?
+        @result[name] = 'down'
+      else
+        if pids.detect {|p| `kill -0 #{p}; echo $?`.chomp.to_i != 0}
+          @result[name] = 'down'
+        else
+          @result[name] = 'up'
+        end
+      end
+    end
+  end
+end

data/lib/ey-flex/bucket_minder.rb

@@ -0,0 +1,119 @@
+module AWS::S3
+  class S3Object
+    def <=>(other)
+      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
+    end
+  end
+end
+
+require 'open-uri'
+
+module EY
+
+  class BucketMinder
+    
+    def initialize(opts={})
+      AWS::S3::Base.establish_connection!(
+          :access_key_id     => opts[:aws_secret_id],
+          :secret_access_key => opts[:aws_secret_key]
+        )
+      @instance_id = opts[:instance_id]
+      @type = opts[:type]
+      @env  = opts[:env]
+      @opts = opts
+      opts[:extension] ||= "tgz"
+      @keep = opts[:keep]
+      @name = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.#{@type}.#{opts[:extension]}"
+    end
+    
+    def bucket
+      @bucket ||= begin
+        buck = "#{@env}-#{@type}-#{instance_id}-#{Digest::SHA1.hexdigest(@opts[:aws_secret_id])[0..6]}"
+          begin
+            AWS::S3::Bucket.create buck
+          rescue AWS::S3::ResponseError
+          end
+        buck
+      end
+    end
+    
+    def instance_id
+      @instance_id ||= open("http://169.254.169.254/latest/meta-data/instance-id").read
+    end
+    
+    def upload_object(file)
+      AWS::S3::S3Object.store(
+         @name,
+         open(file),
+         bucket,
+         :access => :private
+      )
+      FileUtils.rm file
+      puts "successful upload: #{@name}"
+      true
+    end
+    
+    def download(index, printer = false)
+      obj =  list[index.to_i]
+      puts "downloading: #{obj}" if printer
+      File.open(obj.key, 'wb') do |f|
+        print "." if printer
+        obj.value {|chunk| f.write chunk }
+      end
+      puts if printer
+      puts "finished" if printer
+      obj.key
+    end
+    
+    def cleanup
+      begin
+        list[0...-(@keep)].each do |o| 
+          puts "deleting: #{o.key}"  
+          o.delete
+        end
+        # S3's eventual consistency sometimes causes really weird
+        # failures.
+        # Since cleanup happens every time and will clean up all stale
+        # objects, we can just ignore S3-interaction failures. It'll
+        # work next time.
+      rescue AWS::S3::S3Exception, AWS::S3::Error
+        nil
+      end
+    end
+    
+    def get_current
+      name = download(list.size - 1)
+      File.expand_path(name)
+    end
+    
+    def clear_bucket
+      list.each do |o|
+        puts "deleting: #{o.key}" 
+        o.delete
+      end  
+    end
+    
+    def rollback
+      o = list.last
+      puts "rolling back: #{o.key}"  
+      o.delete
+    end
+    
+    def empty?
+      list.empty?
+    end
+    
+    def list(printer = false)
+      objects = AWS::S3::Bucket.objects(bucket).sort
+      puts "listing bucket #{bucket}" if printer && !objects.empty?
+      if printer
+        objects.each_with_index do |b,i|
+          puts "#{i}:#{@env} #{b.key}"
+        end
+      end    
+      objects
+    end
+    
+  end
+  
+end    

data/lib/ey-flex/ey-api.rb

@@ -0,0 +1,21 @@
+module EyApi
+  def call_api(path, opts={})
+    JSON.parse(@rest["/api/#{path}"].post(@keys.merge(opts), {"Accept" => "application/json"}))
+  rescue RestClient::RequestFailed => e
+    case e.http_code
+    when 503
+      sleep 10 # Nanite, save us...
+      retry
+    else
+      raise "API call to Engine Yard failed with status #{e.http_code}."
+    end
+  end
+  
+  def get_envs
+    @_envs ||= call_api("environments")
+  end
+  
+  def get_json(instance_id)
+    call_api("json_for_instance", :instance_id => instance_id)
+  end
+end

data/lib/ey-flex/mysql_database.rb

@@ -0,0 +1,23 @@
+module EY::Flex
+  class MysqlDatabase < DatabaseEngine
+    register_as 'mysql'
+
+    def dump_database(name, io)
+      single_transaction = db_has_myisam?(name) ? '' : '--single-transaction'
+      Open4.spawn ["mysqldump -u#{dbuser} #{password_option} #{single_transaction} #{name} | gzip -c"], :stdout => io
+    end
+
+    def db_has_myisam?(name)
+      query = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{name}' AND engine='MyISAM' LIMIT 1;"
+      %x{mysql -u #{dbuser} #{password_option} -N -e"#{query}"}.strip == '1'
+    end
+
+    def restore_database(name, io)
+      Open4.spawn ["gzip -dc | mysql -u#{dbuser} #{password_option} #{name}"], :stdin => io
+    end
+
+    def password_option
+      dbpass.blank? ? "" : "-p'#{dbpass}'"
+    end
+  end
+end

data/lib/ey-flex/postgresql_database.rb

@@ -0,0 +1,14 @@
+module EY::Flex
+  class PostgresqlDatabase < DatabaseEngine
+    register_as 'postgresql'
+
+    def dump_database(name, io)
+      Open4.spawn ["PGPASSWORD='#{dbpass}' pg_dump -h localhost --clean --no-owner --no-privileges -U#{dbuser} #{name}"], :stdout => io
+    end
+
+    def restore_database(name, io)
+      Open4.spawn ["PGPASSWORD='#{dbpass}' createdb -h localhost -U#{dbuser} #{name}"], :stdin => io, :raise => false
+      Open4.spawn ["PGPASSWORD='#{dbpass}' psql -h localhost -U#{dbuser} #{name}"], :stdin => io
+    end
+  end
+end

data/lib/ey-flex/snapshot_minder.rb

@@ -0,0 +1,205 @@
+module EY
+  class SnapshotMinder
+    def self.run(args)
+      defaults = {:config => '/etc/.mysql.backups.yml',
+                 :command => :list_snapshots,
+                 :keep => 5}
+
+      options = {}
+      # Build a parser for the command line arguments
+      opts = OptionParser.new do |opts|
+        opts.version = "0.0.1"
+
+        opts.banner = "Usage: ey-snapshots [-flag] [argument]"
+        opts.define_head "ey-snapshots: managing your snapshots..."
+        opts.separator '*'*80
+        
+        opts.on("-l", "--list-snapshots", "list snapshots") do 
+          options[:command] = :list_snapshots
+        end
+
+        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+          options[:config] = config
+        end
+        
+        opts.on("-i", "--instance-id ID", "specify the instance id to work with(only needed if you are running this from ourside of ec2)") do |iid|
+          options[:instance_id] = iid
+        end
+        
+        
+        opts.on("--snapshot", "take snapshots of both of your volumes(only runs on your ec2 instance)") do
+          options[:command] = :snapshot_volumes
+        end
+
+        
+      end
+
+      opts.parse!(args)
+
+      ey = nil
+      if File.exist?(config = File.expand_path(defaults[:config]))
+        ey = new(options = defaults.merge(YAML::load(File.read(config))).merge(options))
+      else
+        puts"You need to have an /etc/.mysql.backups.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c .mysql.backups.yml"
+        exit 1
+      end
+
+      ey.send(options[:command])
+      ey.clean_snapshots(options[:keep])
+    end
+
+    def initialize(opts={})
+      @opts = opts
+      @instance_id = opts[:instance_id]
+      @db = Mysql.new('root', opts[:dbpass], opts[:lock_wait_timeout]) rescue nil
+      @ec2 = RightAws::Ec2.new(opts[:aws_secret_id], opts[:aws_secret_key])
+      get_instance_id
+      find_volume_ids
+    end
+    
+    def find_volume_ids
+      @volume_ids = {}
+      @ec2.describe_volumes.each do |volume|
+        if volume[:aws_instance_id] == @instance_id
+          if volume[:aws_device] == "/dev/sdz1"
+            @volume_ids[:data] = volume[:aws_id]
+          elsif volume[:aws_device] == "/dev/sdz2"
+            @volume_ids[:db] = volume[:aws_id]
+          end    
+        end
+      end
+      puts("Volume IDs are #{@volume_ids.inspect}")
+      @volume_ids
+    end
+    
+    def list_snapshots
+      @snapshot_ids = {}
+      @ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+        @volume_ids.each do |mnt, vol|
+          if snapshot[:aws_volume_id] == vol
+            (@snapshot_ids[mnt] ||= []) << snapshot[:aws_id]
+          end  
+        end
+      end
+      puts("Snapshots #{@snapshot_ids.inspect}")
+      @snapshot_ids
+    end
+    
+    def clean_snapshots(keep=5)
+      list_snapshots
+      @snapshot_ids.each do |mnt, ids|
+        snaps = []
+        @ec2.describe_snapshots(ids).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+          snaps << snapshot
+        end
+        (snaps[keep..-1]||[]).each do |snapshot|
+          puts "deleting snapshot of /#{mnt}: #{snapshot[:aws_id]}"
+          @ec2.delete_snapshot(snapshot[:aws_id])
+        end
+      end
+      list_snapshots
+    end
+    
+    def snapshot_volumes
+      snaps = []
+      @volume_ids.each do |vol, vid|
+        case vol
+        when :data
+          sync_filesystem_buffers
+          snaps << create_snapshot(vid)
+        when :db
+          @db.flush_tables_with_read_lock
+          puts("Read lock acquired.  Writing master binlog info to #{@master_status_file} and syncing filesystem buffers.")
+          @db.write_master_status
+          sync_filesystem_buffers
+          snaps << create_snapshot(vid)
+          @db.unlock_tables
+        end
+      end
+      snaps
+    end
+    
+    def get_instance_id
+      return @instance_id if @instance_id
+      
+      open('http://169.254.169.254/latest/meta-data/instance-id') do |f|
+        @instance_id = f.gets
+      end
+      raise "Cannot find instance id!" unless @instance_id
+      puts("Instance ID is #{@instance_id}")
+      @instance_id
+    end
+    
+    def sync_filesystem_buffers
+      sync_cmd = "sync && sync && sync"
+      system(sync_cmd)
+    end
+    
+    def create_snapshot(volume_id)
+      snap = @ec2.create_snapshot(volume_id)
+      puts("Created snapshot of #{volume_id} as #{snap[:aws_id]}")
+      snap
+    end
+    
+  end
+  
+  class Mysql
+    
+    attr_accessor :dbh
+    
+    def initialize(username, password, lock_wait_timeout)
+      @username = username
+      @password = password
+      @read_lock_pid = nil
+      @lock_wait_timeout = lock_wait_timeout.nil? ? 5 : lock_wait_timeout
+      @master_status_file = "/db/mysql/.snapshot_backup_master_status.txt"
+    end
+    
+    def waiting_read_lock_thread
+      thread_cmd = "mysql -p#{@password} -u #{@username} -N -e 'show full processlist;' | grep 'flush tables with read lock' | awk '{print $1}'" 
+      %x{#{thread_cmd}}
+    end
+    
+    def write_master_status
+      master_status_cmd = "mysql -p#{@password} -u #{@username} -e'SHOW MASTER STATUS\\G' > #{@master_status_file}"
+      system(master_status_cmd)
+    end
+    
+    def flush_tables_with_read_lock
+      puts("Requesting read lock for snapshot.")
+      
+      pipe = IO.popen("mysql -u #{@username} -p#{@password}", 'w')
+      @read_lock_pid = pipe.pid
+      
+      pipe.puts('flush tables with read lock;')
+      sleep(@lock_wait_timeout)
+      
+      if (thread_id = waiting_read_lock_thread) != ''
+        Process.kill('TERM', @read_lock_pid)
+        
+        # after killing the process the mysql thread is still hanging out, need to kill it directly
+        kill_thread_cmd = "mysql -u #{@username} -p#{@password} -e'kill #{thread_id};'" 
+        system(kill_thread_cmd)
+        raise "Read lock not acquired after #{@lock_wait_timeout} second timeout.  Killed request and aborting backup."
+      end
+      
+      true
+    end
+    
+    def unlock_tables
+      puts("Unlocking tables")
+      # technically we don't actually have to do anything here since the spawned
+      # process that has the read lock will die with this one but it doesn't hurt
+      # to be safe
+      Process.kill('TERM', @read_lock_pid)
+      true
+    end
+    
+    def disconnect
+      puts("Disconnecting from MySQL")
+      @dbh.disconnect
+    end
+    
+  end
+  
+end 

data/lib/ey-flex/stonith.rb

@@ -0,0 +1,190 @@
+module EY
+  class Log
+    def self.write(str)
+      puts str
+      File.open("/root/ey-monitor2.log", "a") do |f|
+        f.write("#{str}\n")
+      end
+    end
+  end
+  
+  class Stonith
+    include EyApi
+
+    def self.run
+      opts = YAML::load(File.read("/etc/.mysql.backups.yml"))
+      opts.merge!(YAML::load(File.read("/etc/.ey-cloud.yml")))
+      EventMachine.run {
+        EY::Stonith.new(opts.merge(:heartbeat => 10))
+      }
+    end
+
+    def initialize(opts={})
+      Log.write "Starting up"
+      @opts = opts
+      @rest = RestClient::Resource.new(opts[:api])
+      @keys = {:aws_secret_id => @opts[:aws_secret_id], :aws_secret_key => @opts[:aws_secret_key]}
+      @bad_checks = 0
+      @seen_good_check = false
+      @ec2 = RightAws::Ec2.new(@opts[:aws_secret_id], @opts[:aws_secret_key])
+      @taking_over = false
+      get_local_json
+      get_master_from_json
+      setup_traps
+      start
+      am_i_master?
+    end
+    
+    def setup_traps
+      trap("HUP") { cancel_master_check_timer; Log.write "timer canceled, not monitoring until you wake me up again"}
+      trap("USR1") { EM.add_timer(600) { setup_master_check_timer unless am_i_master? }; Log.write "woke up, starting monitoring again in 10 minutes"}
+    end
+    
+    def get_mysql_handle
+      DBI.connect("DBI:Mysql:engineyard:#{@json['db_host']}", 'root', @opts[:dbpass])
+    end
+    
+    def try_lock(nodename)
+      Log.write("Trying to grab the lock for: #{nodename}")
+      db = get_mysql_handle
+      db.execute("begin")
+      res = db.execute("select master_lock from locks for update")
+      master = res.fetch[0]
+      res.finish
+      got_lock = false
+      if master == @master
+        got_lock = true
+        @master = "http://#{private_dns_name}/haproxy/monitor"
+        db.do("update locks set master_lock = '#{@master}'")
+      else
+        # new master, don't start monitoring till it comes up
+        @seen_good_check = false
+        @taking_over = false
+        @master = master
+        Log.write("Failed to grab lock, relenting: #{nodename}\nmaster is: #{@master}")
+        EM.add_timer(600) { Log.write "restarting monitoring"; setup_master_check_timer }
+      end
+      db.do("commit")
+      db.disconnect
+      got_lock
+    end
+    
+    def start
+      setup_master_check_timer
+      EM.add_periodic_timer(300) { get_local_json }
+    end
+    
+    def setup_master_check_timer
+      cancel_master_check_timer
+      unless self_monitor_url == @master
+        @check_master_timer = EventMachine::PeriodicTimer.new(@opts[:heartbeat]) { check_master }
+      end
+    end
+    
+    def cancel_master_check_timer
+      @check_master_timer && @check_master_timer.cancel
+    end
+    
+    def check_master
+      http = EventMachine::HttpRequest.new(@master).get :timeout => 10
+
+      http.callback {
+        unless http.response_header.status == 200
+          take_over_as_master
+        else
+          @seen_good_check = true
+          @bad_checks = 0
+        end
+        http.response_header.status
+      }
+      http.errback { |msg, err|
+        take_over_as_master
+      }
+    end
+    
+    def take_over_as_master
+      Log.write("Got a bad check: seen good check is #{@seen_good_check.inspect}")
+      @bad_checks += 1
+      if @bad_checks > 5 && @seen_good_check && !@taking_over
+        Log.write "I'm trying to take over!"
+        @taking_over = true
+        cancel_master_check_timer
+        if try_lock(private_dns_name)
+          Log.write("I got the lock!")
+          steal_ip
+          unless notify_awsm
+            timer = EventMachine::PeriodicTimer.new(5) { timer.cancel if notify_awsm }
+          end
+        end
+      end
+    end
+    
+    def am_i_master?
+      res = false
+      @ec2.describe_addresses.each do |desc|
+        res = true if (desc[:public_ip] == public_ip && desc[:instance_id] == instance_id)
+      end
+      if res
+        db = get_mysql_handle
+        db.execute("begin")
+        rows = db.execute("select master_lock from locks for update")
+        master = rows.fetch[0]
+        rows.finish
+        db.do("update locks set master_lock = '#{self_monitor_url}'")
+        @master = self_monitor_url
+        cancel_master_check_timer
+        db.do("commit")
+      end
+      res
+    end
+    
+    def notify_awsm
+      Log.write "Notifying awsm that I won"
+      res = call_api("promote_instance_to_master", :instance_id => instance_id)
+      case res['status']
+      when 'ok'
+        true
+      when 'already_promoted'
+        am_i_master?
+        EM.add_timer(600) { setup_master_check_timer unless am_i_master? }
+        true
+      else
+        false
+      end
+    end
+    
+    def instance_id
+      @instance_id ||= open("http://169.254.169.254/latest/meta-data/instance-id").read
+    end
+    
+    def private_dns_name
+      @private_dns_name ||= open("http://169.254.169.254/latest/meta-data/local-hostname").read
+    end
+    
+    def self_monitor_url
+      "http://#{private_dns_name}/haproxy/monitor"
+    end
+    
+    def steal_ip
+      if @ec2.disassociate_address(public_ip)
+        @ec2.associate_address(instance_id, public_ip)
+      end
+    end
+    
+    def public_ip
+      @public_ip ||= @json['master_app_server']['public_ip']
+    end
+    
+    def get_local_json
+      @json = JSON.parse(File.read("/etc/chef/dna.json"))
+    end
+    
+    def get_master_from_json
+      if host = @json['master_app_server']['private_dns_name']
+        @master = "http://#{host}/haproxy/monitor"
+      end
+    end
+    
+  end
+end
+

data/lib/ey-flex/version.rb

@@ -0,0 +1,3 @@
+module EY::CloudServer
+  VERSION = '1.0.1'
+end

data/lib/ey-flex.rb

@@ -0,0 +1,37 @@
+require 'json'
+require 'aws/s3'
+require 'date'
+require 'digest'
+require 'net/http'
+require 'fileutils'
+require 'eventmachine'
+require 'em-http'
+require 'json/ext'
+require 'right_aws'
+require 'open-uri'
+require 'rest_client'
+require 'dbi'
+require 'zlib'
+require 'stringio'
+require 'yaml'
+require "optparse"
+require 'open4'
+
+lib_dir = File.expand_path(__FILE__ + '/../ey-flex')
+
+module EY
+  module Flex
+    class Error < StandardError; end
+  end
+  module CloudServer; end
+end
+
+require lib_dir + '/big-brother'
+require lib_dir + '/backups'
+require lib_dir + '/bucket_minder'
+require lib_dir + '/ey-api'
+require lib_dir + '/mysql_database'
+require lib_dir + '/postgresql_database'
+require lib_dir + '/snapshot_minder'
+require lib_dir + '/stonith'
+require lib_dir + '/version'

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: ey_cloud_server
+version: !ruby/object:Gem::Version 
+  version: 1.0.1
+platform: ruby
+authors: 
+- Ezra Zygmuntowicz
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-20 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: open4
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: igrigorik-em-http-request
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rest-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Server side components for Engine Yard's cloud
+email: awsmdev@engineyard.com
+executables: 
+- eybackup
+- ey-snapshots
+- ey-monitor
+- ey-agent
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- LICENSE
+- README.rdoc
+- lib/ey-flex/backups.rb
+- lib/ey-flex/big-brother.rb
+- lib/ey-flex/bucket_minder.rb
+- lib/ey-flex/ey-api.rb
+- lib/ey-flex/mysql_database.rb
+- lib/ey-flex/postgresql_database.rb
+- lib/ey-flex/snapshot_minder.rb
+- lib/ey-flex/stonith.rb
+- lib/ey-flex/version.rb
+- lib/ey-flex.rb
+has_rdoc: true
+homepage: http://engineyard.com/cloud
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Server side components for Engine Yard's cloud
+test_files: []
+