ey-flex-test 0.3.3

data/lib/mysql_slave.rb

@@ -0,0 +1,22 @@
+
+require File.join(File.dirname(__FILE__), 'snapshot_minder')
+require 'json'
+module EY
+  class MysqlSlave
+    
+    def initialize
+      @dna = ::JSON.parse(IO.read('/etc/chef/dna.json'))
+    end
+    
+    def bootstrap
+      puts("reading in /db/mysql/master_status.json to get position in the binlog")
+      master_status = ::JSON.parse(IO.read('/db/mysql/master_status.json'))
+      master_status["master_host"] = @dna['db_host']
+      master_status["master_user"] = 'replication'
+      master_status["master_password"] = @dna['users'].first['password']
+      @db = ::EY::Mysql.new('root', @dna['users'].first['password'])
+      @db.change_master(master_status)
+      @db.slave_start
+    end
+  end
+end

data/lib/postgresql_backup.rb

@@ -0,0 +1,31 @@
+module EyBackup
+  class PostgresqlBackup < MysqlBackup
+    def backup_database(database)
+      full_path_to_backup = "#{self.backup_dir}/#{database}.#{@tmpname}"
+      posgrecmd = "PGPASSWORD='#{@dbpass}' pg_dump --clean --no-owner --no-privileges -U#{@dbuser} #{database} | gzip - > #{full_path_to_backup}"
+      if system(posgrecmd)
+        AWS::S3::S3Object.store(
+           "/#{@id}.#{database}/#{database}.#{@tmpname}",
+           open(full_path_to_backup),
+           @bucket,
+           :access => :private
+        )
+        FileUtils.rm full_path_to_backup
+        puts "successful backup: #{database}.#{@tmpname}"
+      else
+        raise "Unable to dump database#{database} wtf?"
+      end
+    end
+
+    def restore(index)
+      name = download(index)
+      db = name.split('.').first
+      cmd = "gunzip -c #{name} | PGPASSWORD='#{@dbpass}' psql -U#{@dbuser} #{db}"
+      if system(cmd)
+        puts "successfully restored backup: #{name}"
+      else
+        puts "FAIL"
+      end    
+    end
+  end
+end

data/lib/snapshot_minder.rb

@@ -0,0 +1,157 @@
+require 'right_aws'
+require 'open-uri'
+require 'dbi'
+require 'json'
+module EY
+  class SnapshotMinder
+    def initialize(opts={})
+      @opts = opts
+      @instance_id = opts[:instance_id]
+      @db = Mysql.new('root', opts[:dbpass]) rescue nil
+      @ec2 = RightAws::Ec2.new(opts[:aws_secret_id], opts[:aws_secret_key])
+      get_instance_id
+      find_volume_ids
+    end
+    
+    def find_volume_ids
+      @volume_ids = {}
+      @ec2.describe_volumes.each do |volume|
+        if volume[:aws_instance_id] == @instance_id
+          if volume[:aws_device] == "/dev/sdz1"
+            @volume_ids[:data] = volume[:aws_id]
+          elsif volume[:aws_device] == "/dev/sdz2"
+            @volume_ids[:db] = volume[:aws_id]
+          end    
+        end
+      end
+      puts("Volume IDs are #{@volume_ids.inspect}")
+      @volume_ids
+    end
+    
+    def list_snapshots
+      @snapshot_ids = {}
+      @ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+        @volume_ids.each do |mnt, vol|
+          if snapshot[:aws_volume_id] == vol
+            (@snapshot_ids[mnt] ||= []) << snapshot[:aws_id]
+          end  
+        end
+      end
+      puts("Snapshots #{@snapshot_ids.inspect}")
+      @snapshot_ids
+    end
+    
+    def clean_snapshots(keep=5)
+      list_snapshots
+      @snapshot_ids.each do |mnt, ids|
+        snaps = []
+        @ec2.describe_snapshots(ids).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
+          snaps << snapshot
+        end
+        (snaps[keep..-1]||[]).each do |snapshot|
+          puts "deleting snapshot of /#{mnt}: #{snapshot[:aws_id]}"
+          @ec2.delete_snapshot(snapshot[:aws_id])
+        end
+      end
+      list_snapshots
+    end
+    
+    def snapshot_volumes
+      snaps = []
+      @volume_ids.each do |vol, vid|
+        case vol
+        when :data
+          snaps << create_snapshot(vid)
+        when :db
+          @db.flush_tables_with_read_lock
+          master_status = @db.show_master_status
+          ms_json = File.open('/db/mysql/master_status.json', "w")
+          JSON.dump(master_status, ms_json)
+          ms_json.fsync
+          ms_json.close
+          snaps << create_snapshot(vid)
+          @db.unlock_tables
+        end
+      end
+      snaps
+    end
+    
+    def get_instance_id
+      return @instance_id if @instance_id
+      
+      open('http://169.254.169.254/latest/meta-data/instance-id') do |f|
+        @instance_id = f.gets
+      end
+      raise "Cannot find instance id!" unless @instance_id
+      puts("Instance ID is #{@instance_id}")
+      @instance_id
+    end
+    
+    
+    def create_snapshot(volume_id)
+      snap = @ec2.create_snapshot(volume_id)
+      puts("Created snapshot of #{volume_id} as #{snap[:aws_id]}")
+      snap
+    end
+    
+  end
+  
+  class Mysql
+    
+    attr_accessor :dbh
+    
+    def initialize(username, password)
+      @username = username
+      @password = password
+      puts("Connecting to MySQL")
+      @dbh = DBI.connect("DBI:Mysql:mysql", username, password)
+    end
+    
+    def show_master_status
+      status = Hash.new
+      @dbh.select_all("show master status") do |row|
+        row.each_with_name do |v, column|
+          status[column] = v
+        end
+      end
+      puts("Master status: #{status.inspect}")
+      status
+    end
+    
+    def change_master(master_status)
+      command = "CHANGE MASTER TO"
+      command << " MASTER_HOST='#{master_status['master_host']}',"
+      command << " MASTER_USER='#{master_status['master_user']}',"
+      command << " MASTER_PASSWORD='#{master_status['master_password']}',"
+      command << " MASTER_LOG_FILE='#{master_status['File']}',"
+      command << " MASTER_LOG_POS=#{master_status['Position']}"
+      puts(command)
+      @dbh.do(command)
+      puts("Master is now #{master_status['master_host']} at #{master_status['File']} pos #{master_status['Position']}")
+    end
+    
+    def slave_start
+      @dbh.do("slave start")
+      puts("Slave started")
+    end
+    
+    def flush_tables_with_read_lock
+      puts("Flushing tables with read lock")
+      @dbh.do("flush tables with read lock")
+      true
+    end
+    
+    def unlock_tables
+      puts("Unlocking tables")
+      @dbh.do("unlock tables")
+      true
+    end
+    
+    def disconnect
+      puts("Disconnecting from MySQL")
+      @dbh.disconnect
+    end
+    
+  end
+  
+end 

data/lib/stonith.rb

@@ -0,0 +1,194 @@
+require 'rubygems' 
+require 'eventmachine'
+# gem install igrigorik-em-http-request --source http://gems.github.com
+gem 'igrigorik-em-http-request'
+require 'em-http'
+require 'json/ext'
+#require 'ey-api'
+require 'right_aws'
+require 'open-uri'
+require 'rest_client'
+require 'dbi'
+require 'ey-api'
+
+module EY
+  class Log
+    def self.write(str)
+      puts str
+      File.open("/root/ey-monitor2.log", "a") do |f|
+        f.write("#{str}\n")
+      end
+    end
+  end
+  
+  class Stonith
+    include EyApi
+    def initialize(opts={})
+      Log.write "Starting up"
+      @opts = opts
+      @rest = RestClient::Resource.new(opts[:api])
+      @keys = {:aws_secret_id => @opts[:aws_secret_id], :aws_secret_key => @opts[:aws_secret_key]}
+      @bad_checks = 0
+      @seen_good_check = false
+      @ec2 = RightAws::Ec2.new(@opts[:aws_secret_id], @opts[:aws_secret_key])
+      @taking_over = false
+      get_local_json
+      get_master_from_json
+      setup_traps
+      start
+      am_i_master?
+    end
+    
+    def setup_traps
+      trap("HUP") { cancel_master_check_timer; Log.write "timer canceled, not monitoring until you wake me up again"}
+      trap("USR1") { EM.add_timer(600) { setup_master_check_timer unless am_i_master? }; Log.write "woke up, starting monitoring again in 10 minutes"}
+    end
+    
+    def get_mysql_handle
+      DBI.connect("DBI:Mysql:engineyard:#{@json['db_host']}", 'root', @opts[:dbpass])
+    end
+    
+    def try_lock(nodename)
+      Log.write("Trying to grab the lock for: #{nodename}")
+      db = get_mysql_handle
+      db.execute("begin")
+      res = db.execute("select master_lock from locks for update")
+      master = res.fetch[0]
+      res.finish
+      got_lock = false
+      if master == @master
+        got_lock = true
+        @master = "http://#{private_dns_name}/haproxy/monitor"
+        db.do("update locks set master_lock = '#{@master}'")
+      else
+        # new master, don't start monitoring till it comes up
+        @seen_good_check = false
+        @taking_over = false
+        @master = master
+        Log.write("Failed to grab lock, relenting: #{nodename}\nmaster is: #{@master}")
+        EM.add_timer(600) { Log.write "restarting monitoring"; setup_master_check_timer }
+      end
+      db.do("commit")
+      db.disconnect
+      got_lock
+    end
+    
+    def start
+      setup_master_check_timer
+      EM.add_periodic_timer(300) { get_local_json }
+    end
+    
+    def setup_master_check_timer
+      cancel_master_check_timer
+      unless self_monitor_url == @master
+        @check_master_timer = EventMachine::PeriodicTimer.new(@opts[:heartbeat]) { check_master }
+      end
+    end
+    
+    def cancel_master_check_timer
+      @check_master_timer && @check_master_timer.cancel
+    end
+    
+    def check_master
+      http = EventMachine::HttpRequest.new(@master).get :timeout => 10
+
+      http.callback {
+        unless http.response_header.status == 200
+          take_over_as_master
+        else
+          @seen_good_check = true
+          @bad_checks = 0
+        end
+        http.response_header.status
+      }
+      http.errback { |msg, err|
+        take_over_as_master
+      }
+    end
+    
+    def take_over_as_master
+      Log.write("Got a bad check: seen good check is #{@seen_good_check.inspect}")
+      @bad_checks += 1
+      if @bad_checks > 5 && @seen_good_check && !@taking_over
+        Log.write "I'm trying to take over!"
+        @taking_over = true
+        cancel_master_check_timer
+        if try_lock(private_dns_name)
+          Log.write("I got the lock!")
+          steal_ip
+          unless notify_awsm
+            timer = EventMachine::PeriodicTimer.new(5) { timer.cancel if notify_awsm }
+          end
+        end
+      end
+    end
+    
+    def am_i_master?
+      res = false
+      @ec2.describe_addresses.each do |desc|
+        res = true if (desc[:public_ip] == public_ip && desc[:instance_id] == instance_id)
+      end
+      if res
+        db = get_mysql_handle
+        db.execute("begin")
+        rows = db.execute("select master_lock from locks for update")
+        master = rows.fetch[0]
+        rows.finish
+        db.do("update locks set master_lock = '#{self_monitor_url}'")
+        @master = self_monitor_url
+        cancel_master_check_timer
+        db.do("commit")
+      end
+      res
+    end
+    
+    def notify_awsm
+      Log.write "Notifying awsm that I won"
+      res = call_api("promote_instance_to_master", :instance_id => instance_id)
+      case res['status']
+      when 'ok'
+        true
+      when 'already_promoted'
+        am_i_master?
+        EM.add_timer(600) { setup_master_check_timer unless am_i_master? }
+        true
+      else
+        false
+      end
+    end
+    
+    def instance_id
+      @instance_id ||= open("http://169.254.169.254/latest/meta-data/instance-id").read
+    end
+    
+    def private_dns_name
+      @private_dns_name ||= open("http://169.254.169.254/latest/meta-data/local-hostname").read
+    end
+    
+    def self_monitor_url
+      "http://#{private_dns_name}/haproxy/monitor"
+    end
+    
+    def steal_ip
+      if @ec2.disassociate_address(public_ip)
+        @ec2.associate_address(instance_id, public_ip)
+      end
+    end
+    
+    def public_ip
+      @public_ip ||= @json['master_app_server']['public_ip']
+    end
+    
+    def get_local_json
+      @json = JSON.parse(IO.read("/etc/chef/dna.json"))
+    end
+    
+    def get_master_from_json
+      if host = @json['master_app_server']['private_dns_name']
+        @master = "http://#{host}/haproxy/monitor"
+      end
+    end
+    
+  end
+end
+

data/spec/ey_api_spec.rb

@@ -0,0 +1,74 @@
+$:.unshift File.dirname(__FILE__) + "/../lib/"
+require 'ey'
+require 'fakeweb'
+
+describe "EyApi" do
+  describe "#get_json" do
+    before(:each) do
+      @mock_environment_name = 'myenv'
+
+      FakeWeb.allow_net_connect = false # if it's not here, it doesn't exist
+      FakeWeb.register_uri(:post, "http://example.org/api/environments", :body => JSON.generate({
+        @mock_environment_name => {:id => 1}
+      }))
+    end
+
+    before(:each) do
+      @api = Object.new
+      @api.instance_variable_set("@rest", RestClient::Resource.new('http://example.org'))
+      @api.instance_variable_set("@env", @mock_environment_name)
+      @api.instance_variable_set("@keys", {})
+      @api.extend(EyApi)
+
+      @api.stub!(:sleep)
+    end
+
+    after(:each) do
+      FakeWeb.clean_registry
+    end
+
+    describe "when the JSON is available" do
+      before(:each) do
+        FakeWeb.register_uri(:post, "http://example.org/api/json_for_instance", :body => JSON.generate({:valid => "json"}))
+      end
+
+      it "should pass through exceptions that it receives" do
+        JSON.stub!(:parse).and_raise(Exception)
+        lambda {
+          @api.get_json("i-decafbad")
+        }.should raise_error
+      end
+
+      it "should return a valid JSON structure" do
+        presumably_valid_json = @api.get_json("i-feedface")
+        lambda {
+          JSON[presumably_valid_json]
+        }.should_not raise_error
+      end
+    end
+
+    describe "when the JSON is not yet available" do
+      
+      it "should retry until the 503 goes away" do
+        FakeWeb.register_uri(:post, "http://example.org/api/json_for_instance", [{
+            :status => ["503", "Service Unavailable"],
+            :body => "{}",
+          }, {
+            :status => ["503", "Service Unavailable"],
+            :body => "{}",
+          }, {
+            :status => ["200", "OK"],
+            :body => JSON.generate({:valid => 'json'}),
+          }]
+        )
+
+        lambda {
+          JSON[@api.get_json("i-robot")]
+        }.should_not raise_error
+      end
+      
+    end
+  end
+end
+
+