ey_cloud_server 1.4.5 → 1.4.26

data/bin/binary_log_purge

@@ -100,14 +100,22 @@ def run_query(host, user, password, query)
   if host == '127.0.0.1'
     options = options + ' -P13306'
   end
-  stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"")
+  if query == 'show processlist'
+    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"|grep 'Binlog'")
+  else
+    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"")
+  end
   query_error = stderr.read
   if query_error.length > 0
     log_error "Error caught: #{query_error}"
     test_add_privilege(user, password, query_error)
     exit 0
   end
-  rs = stdout.read
+  result = stdout.read
+  stdin.close
+  stdout.close
+  stderr.close
+  result
 end
 
 # function to test for user privilege

data/bin/ey-snapshots

@@ -1,9 +1,15 @@
 #!/usr/bin/env ruby
+
+# Disable verbose (when running from cron) to squash warnings output by rest_client (1.6.1)
+unless STDOUT.isatty
+  $VERBOSE = nil
+end
+
 require File.dirname(__FILE__) + '/../lib/ey-flex'
 
 begin
   EY::SnapshotMinder.run(ARGV)
 rescue => e
-  EY.notify_error(e)
+  EY.notify_snapshot_error(e)
   raise
 end

data/bin/eybackup

@@ -1,6 +1,18 @@
 #!/usr/bin/env ruby
 
+# Disable verbose (when running from cron) to squash warnings output by rest_client (1.6.1) and also fog (0.4.0)
+unless STDOUT.isatty
+  $VERBOSE = nil
+end
+
 require 'rubygems'
 require File.dirname(__FILE__) + '/../lib/ey_backup'
+require File.dirname(__FILE__) + '/../lib/ey-flex'
+
+begin
+  EY::Backup.run(ARGV)
+rescue => e
+  EY.notify_backup_error(e)
+  raise
+end
 
-EY::Backup.run(ARGV)

data/lib/ey-flex.rb

@@ -14,19 +14,29 @@ require 'stringio'
 require 'yaml'
 require "optparse"
 require 'open4'
+require "ey_enzyme"
 
 lib_dir = File.expand_path(__FILE__ + '/../ey-flex')
 
 module EY
-  def self.notify_error(error)
-    enzyme_api.notify_error("user", error)
+  def self.notify_snapshot_error(error)
+    enzyme_api.notify_error("snapshot", error)
+  end
+
+  def self.notify_backup_error(error)
+    enzyme_api.notify_error("backup", error)
   end
 
   def self.enzyme_api
-    return @enzyme_api if @enzyme_api
-    require 'ey_enzyme'
-    config = YAML.load_file("/etc/engineyard/dracul.yml")
-    @enzyme_api = EY::Enzyme::API.new(config[:api], config[:instance_id], config[:token])
+    @enzyme_api ||= EY::Enzyme::API.new(
+      enzyme_config[:api],
+      enzyme_config[:instance_id],
+      enzyme_config[:token]
+    )
+  end
+
+  def self.enzyme_config
+    @enzyme_config ||= YAML.load_file("/etc/engineyard/dracul.yml")
   end
 
   module Flex
@@ -39,4 +49,5 @@ require lib_dir + '/big-brother'
 require lib_dir + '/bucket_minder'
 require lib_dir + '/ey-api'
 require lib_dir + '/snapshot_minder'
-require lib_dir + '/version'
+require lib_dir + '/ec2'
+require lib_dir + '/../ey_cloud_server/version'

data/lib/ey-flex/big-brother.rb

@@ -1,7 +1,7 @@
 module EY
   class BigBrother
-    def self.check
-      json = JSON.parse(File.read('/etc/chef/dna.json'))
+    def self.check(chef_dna = '/etc/chef/dna.json')
+      json = JSON.parse(File.read(chef_dna))
 
       # {'skip':[
       #   'mysqld'

data/lib/ey-flex/bucket_minder.rb

@@ -1,194 +1,80 @@
-module AWS::S3
-  class S3Object
-    def <=>(other)
-      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
-    end
-  end
-end
-
 require 'open-uri'
 
 module EY
-
   class BucketMinder
-    attr_accessor :bucket, :name
-    
-    def initialize(opts={})
-      AWS::S3::Base.establish_connection!(
-          :access_key_id     => opts[:aws_secret_id],
-          :secret_access_key => opts[:aws_secret_key]
-        )
-      @instance_id = opts[:instance_id]
-      @type = opts[:type]
-      @env  = opts[:env]
-      @opts = opts
-      @bucket = find_bucket(opts[:bucket])
-      
-      opts[:extension] ||= "tgz"
-      @keep = opts[:keep]
-      @name = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.#{@type}.#{opts[:extension]}"
-    end
-    
-    def find_bucket(buck="#{@env}-#{@type}-#{@instance_id}-#{Digest::SHA1.hexdigest(@opts[:aws_secret_id])[0..6]}")
-      begin
-        begin
-            AWS::S3::Bucket.find(buck)
-          rescue AWS::S3::NoSuchBucket
-            begin
-              AWS::S3::Bucket.create(buck)
-            rescue AWS::S3::ResponseError
-            end
-          end          
-        buck
-      end
-    end
-    
-    def instance_id
-      @instance_id ||= open("http://169.254.169.254/latest/meta-data/instance-id").read
-    end
-    
-    def upload_object(file, printer = true)
-      AWS::S3::S3Object.store(
-         @name,
-         File.open(file),
-         bucket,
-         :access => :private
-      )
-      FileUtils.rm file
-      puts "successful upload: #{@name}" if printer
-      true
-    end
-    
-    def remove_object(key, printer = false)
-      begin
-        AWS::S3::S3Object.delete(key, @bucket)
-        puts "Deleting #{key}" if printer
-        # S3's eventual consistency sometimes causes really weird
-        # failures.
-        # Since cleanup happens every time and will clean up all stale
-        # objects, we can just ignore S3-interaction failures. It'll
-        # work next time.
-      rescue AWS::S3::S3Exception, AWS::S3::Error
-        nil
-      end
+    def initialize(secret_id, secret_key, bucket_name, region = 'us-east-1')
+      @s3          = Fog::Storage.new(:provider => 'AWS',:aws_access_key_id => secret_id, :aws_secret_access_key => secret_key, :region => region)
+      @region      = region
+      @bucket_name = bucket_name || "ey-backup-#{Digest::SHA1.hexdigest(secret_id)[0..11]}"
+
+      setup_bucket
     end
-    
-    def retrieve_object(key, printer = false)
-      puts "Retrieving #{key}" if printer
-      obj = AWS::S3::S3Object.find(key, @bucket)
+
+    attr_reader :bucket_name
+
+    def bucket
+      @bucket ||= @s3.directories.get(@bucket_name)
     end
-    
-    def download(index, printer = false)
-      obj =  list[index.to_i]
-      puts "downloading: #{obj}" if printer
-      File.open(obj.key, 'wb') do |f|
-        print "." if printer
-        obj.value {|chunk| f.write chunk }
-      end
-      puts if printer
-      puts "finished" if printer
-      obj.key
+
+    def files
+      bucket.files
     end
 
-    def stream(key)
-      AWS::S3::S3Object.stream(key, @bucket) do |chunk|
-        yield chunk
+    def setup_bucket
+      unless bucket
+        @s3.directories.create(s3_params(:key => @bucket_name))
       end
     end
-    
-    def cleanup
-      begin
-        list[0...-(@keep)].each do |o| 
-          puts "deleting: #{o.key}"  
-          o.delete
-        end
-        # S3's eventual consistency sometimes causes really weird
-        # failures.
-        # Since cleanup happens every time and will clean up all stale
-        # objects, we can just ignore S3-interaction failures. It'll
-        # work next time.
-      rescue AWS::S3::S3Exception, AWS::S3::Error
-        nil
+
+    def s3_params(params = {})
+      return params if @region == 'us-east-1'
+      if @region == 'eu-west-1'
+        params.merge({:location => 'EU'})
+      else
+        params.merge({:location => @region})
       end
     end
-    
-    def get_current
-      name = download(list.size - 1)
-      File.expand_path(name)
-    end
-    
-    # Removes all of the items in the current bucket
-    def clear_bucket
-      list.each do |o|
-        puts "deleting: #{o.key}"
-        o.delete
-      end  
-    end
-    
-    # Deletes the most recent file from the bucket
-    def rollback
-      o = list.last
-      puts "rolling back: #{o.key}"  
-      o.delete
+
+    def remove_object(key)
+      @s3.delete_object(bucket.key, key)
     end
-    
-    def empty?
-      list.empty?
+
+    def stream(key, &block)
+      files.get(key, &block)
     end
-    
-    def list(directive = false)
-      if directive.is_a? Hash
-        prefix = directive[:prefix]
-        printer = directive[:print] ||= false
-        listing = AWS::S3::Bucket.objects(bucket, :prefix => prefix).flatten.sort
-        objects = s3merge(listing)
-      elsif directive.is_a? String
-        printer = directive
-        objects = AWS::S3::Bucket.objects(bucket).sort
-      else
-        puts "Input to list method must be Hash or String not: #{printer.class}"
-        exit
-      end
 
-      return objects if objects.empty?
-      
-      puts "listing bucket #{bucket}" if printer && !objects.empty?
-      
-      if printer
-        objects.each_with_index do |b, i|
-          if b.is_a? Hash
-            puts "#{i}:#{@env} #{b[:name]}"
-          else
-            puts "#{i}:#{@env} #{b.key}"
-          end
-        end
-      end    
-      objects
+    def list(prefix)
+      listing = files.all(:prefix => prefix)
+      s3merge(listing)
     end
-    
+
     # Merge s3 file listing to work with split files with naming of *.part\d\d
     def s3merge(list)
       return list if list.empty?
-      bucketfiles=Array.new()
+      distinct_files=Array.new()
+
       list.each do |item|
         fname = item.key.gsub(/.part\d+$/,'')
         match = false
-        bucketfiles.each_with_index do |b, i|
+        distinct_files.each_with_index do |b, i|
           if b[:name] == fname
-            bucketfiles[i][:keys] << item.key
+            distinct_files[i][:keys] << item.key
             match = true
           end
         end
-        
+
         if not match
           path = Array.new()
           path << item.key
           file = {:name => fname, :keys => path}
-          bucketfiles << file
+          distinct_files << file
         end
       end
-      bucketfiles
+      distinct_files
+    end
+
+    def put(filename, contents)
+      files.create(:key => filename, :body => contents)
     end
-    
   end
-end    
+end

data/lib/ey-flex/ec2.rb

@@ -0,0 +1,17 @@
+module EY
+  class EC2
+    def initialize(opts = {})
+      @ec2 = RightAws::Ec2.new(opts[:aws_secret_id], opts[:aws_secret_key], :logger => Logger.new("/dev/null"))
+    end
+
+    def method_missing(method, *a, &b)
+      @ec2.send(method, *a, &b)
+    rescue RightAws::AwsError => e
+      retries ||= 10
+      retries -= 1
+      raise e if retries == 0
+      sleep 30
+      retry
+    end
+  end
+end

data/lib/ey-flex/snapshot_minder.rb

@@ -1,11 +1,64 @@
+require 'ey_instance_api_client'
 module EY
   class SnapshotMinder
-    def self.run(args)
-      defaults = {:config => '/etc/.mysql.backups.yml',
-                 :command => :list_snapshots,
-                 :keep => 5}
 
-      options = {}
+    class Backend
+      class Mock
+        def initialize
+          @commands_called = []
+          @raise_count = 0
+          @total_wait_time = 0
+        end
+
+        attr_reader :commands_called, :total_wait_time
+        attr_accessor :raise_count
+
+        def wait(seconds)
+          @total_wait_time += seconds
+        end
+
+        def snapshot
+          if @raise_count > 0
+            @raise_count -= 1
+            raise EY::InstanceAPIClient::Connection::UnexpectedStatus.new("")
+          else
+            @commands_called << "request_snapshot"
+          end
+        end
+
+        def list
+          @commands_called << "list_snapshots"
+          [EY::InstanceAPIClient::Snapshot.new({
+            'state' => 'in progres',
+            'progress' => '0%',
+            'volume_type' => 'db',
+            'snapshot_id' => 12,
+            'created_at' => Time.now.to_s
+            })]
+        end
+      end
+
+      class Real
+        def initialize
+          @snapshots = EY::InstanceAPIClient::Snapshots.new
+        end
+
+        def wait(seconds)
+          sleep(seconds)
+        end
+
+        def snapshot
+          @snapshots.request
+        end
+
+        def list
+          @snapshots.list
+        end
+      end
+    end
+
+    def self.run(args)
+      options = {:command => :list_snapshots}
       # Build a parser for the command line arguments
       opts = OptionParser.new do |opts|
         opts.version = "0.0.1"
@@ -18,16 +71,15 @@ module EY
           options[:command] = :list_snapshots
         end
 
-        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
-          options[:config] = config
+        opts.on("-c", "--config CONFIG", "DEPRECATED, does nothing") do |config|
+          STDERR.puts "WARNING: --config is DEPRECATED and has no effect"
         end
 
-        opts.on("-i", "--instance-id ID", "specify the instance id to work with(only needed if you are running this from ourside of ec2)") do |iid|
-          options[:instance_id] = iid
+        opts.on("-i", "--instance-id ID", "DEPRECATED, does nothing") do |iid|
+          STDERR.puts "WARNING: --instance-id is DEPRECATED and has no effect"
         end
 
-
-        opts.on("--snapshot", "take snapshots of both of your volumes(only runs on your ec2 instance)") do
+        opts.on("--snapshot", "request a snapshot of the volumes attached to this instance") do
           options[:command] = :snapshot_volumes
         end
 
@@ -38,186 +90,56 @@ module EY
       end
 
       opts.parse!(args)
-
-      ey = nil
-      if File.exist?(config = File.expand_path(defaults[:config]))
-        ey = new(options = defaults.merge(YAML::load(File.read(config))).merge(options))
-      else
-        abort "You need to have an /etc/.mysql.backups.yml file with your credentials in it to use this tool.\nOr point it at a yaml file with -c .mysql.backups.yml"
-      end
-
-      ey.send(options[:command])
-      ey.clean_snapshots(options[:keep])
+      new(options).send(options[:command])
     end
 
-    def initialize(opts={})
-      @opts = opts
-      @instance_id = opts[:instance_id]
-      @db = Mysql.new('root', opts[:dbpass], opts[:lock_wait_timeout]) rescue nil
-      @ec2 = RightAws::Ec2.new(opts[:aws_secret_id], opts[:aws_secret_key], :logger => Logger.new("/dev/null"))
-      get_instance_id
-      silence_stream($stderr) { find_volume_ids }
+    def self.backend
+      @backend ||= Backend::Real.new
     end
 
-    def find_volume_ids
-      @volume_ids = {}
-      @ec2.describe_volumes.each do |volume|
-        if volume[:aws_instance_id] == @instance_id
-          if volume[:aws_device] == "/dev/sdz1"
-            @volume_ids[:data] = volume[:aws_id]
-          elsif volume[:aws_device] == "/dev/sdz2"
-            @volume_ids[:db] = volume[:aws_id]
-          end
-        end
-      end
-      say "Volume IDs are #{@volume_ids.inspect}"
-      @volume_ids
+    def self.enable_mock!
+      @backend = Backend::Mock.new
     end
 
-    def list_snapshots
-      @snapshot_ids = {}
-      @ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
-        @volume_ids.each do |mnt, vol|
-          if snapshot[:aws_volume_id] == vol
-            (@snapshot_ids[mnt] ||= []) << snapshot[:aws_id]
-          end
-        end
-      end
-      say "Snapshots #{@snapshot_ids.inspect}"
-      @snapshot_ids
-    end
-
-    def clean_snapshots(keep=5)
-      list_snapshots
-      @snapshot_ids.each do |mnt, ids|
-        snaps = []
-        @ec2.describe_snapshots(ids).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
-          snaps << snapshot
-        end
-        (snaps[keep..-1]||[]).each do |snapshot|
-          say "deleting snapshot of /#{mnt}: #{snapshot[:aws_id]}"
-          @ec2.delete_snapshot(snapshot[:aws_id])
-        end
-      end
-      list_snapshots
+    def initialize(opts={})
+      @opts = opts
     end
 
     def snapshot_volumes
-      snaps = []
-      @volume_ids.each do |vol, vid|
-        case vol
-        when :data
-          sync_filesystem_buffers
-          snaps << create_snapshot(vid)
-        when :db
-          @db.flush_tables_with_read_lock
-          say "Read lock acquired.  Writing master binlog info to #{@master_status_file} and syncing filesystem buffers."
-          @db.write_master_status
-          sync_filesystem_buffers
-          snaps << create_snapshot(vid)
-          @db.unlock_tables
+      retry_count = 0
+      begin
+        SnapshotMinder.backend.snapshot
+        say "Snapshot requested."
+      rescue EY::InstanceAPIClient::Connection::UnexpectedStatus => e
+        retry_count += 1
+        if retry_count <= 10
+          retry_in = backoff(retry_count)
+          say "failed with #{e.inspect}, retrying in #{retry_in} seconds"
+          SnapshotMinder.backend.wait(retry_in)
+          retry
+        else
+          raise e
         end
       end
-      snaps
-    end
-
-    def get_instance_id
-      return @instance_id if @instance_id
-
-      open('http://169.254.169.254/latest/meta-data/instance-id') do |f|
-        @instance_id = f.gets
-      end
-      abort "Cannot find instance id!" unless @instance_id
-      say "Instance ID is #{@instance_id}"
-      @instance_id
     end
 
-    def sync_filesystem_buffers
-      sync_cmd = "sync && sync && sync"
-      system(sync_cmd)
+    def backoff(nth_time)
+      60*nth_time + rand((60 / (nth_time+1)) + 5) #see tests
     end
 
-    def create_snapshot(volume_id)
-      retries = 0
-      begin
-        snap = @ec2.create_snapshot(volume_id)
-      rescue RightAws::AwsError
-        retries += 1
-        raise if retries > 10
-        sleep retries * retries
-        retry
+    def list_snapshots
+      snapshots = SnapshotMinder.backend.list
+      say "#{snapshots.size} Snapshots available:"
+      snapshots.each do |s|
+        say "#{s.id} - #{s.volume} #{s.state} #{s.progress} #{s.created_at}"
       end
-      say "Created snapshot of #{volume_id} as #{snap[:aws_id]}"
-      snap
     end
 
     private
-      def say(msg, newline = true)
-        return if @opts[:quiet]
-        print("#{msg}#{"\n" if newline}")
-      end
-
-      def silence_stream(stream)
-        old_stream = stream.dup
-        stream.reopen("/dev/null")
-        stream.sync = true
-        yield
-      ensure
-        stream.reopen(old_stream)
-      end
-  end
-
-  class Mysql
-
-    attr_accessor :dbh
-
-    def initialize(username, password, lock_wait_timeout)
-      @username = username
-      @password = password
-      @read_lock_pid = nil
-      @lock_wait_timeout = lock_wait_timeout.nil? ? 5 : lock_wait_timeout
-      @master_status_file = "/db/mysql/.snapshot_backup_master_status.txt"
-    end
-
-    def waiting_read_lock_thread
-      thread_cmd = "mysql -p#{@password} -u #{@username} -N -e 'show full processlist;' | grep 'flush tables with read lock' | awk '{print $1}'"
-      %x{#{thread_cmd}}
-    end
-
-    def write_master_status
-      master_status_cmd = "mysql -p#{@password} -u #{@username} -e'SHOW MASTER STATUS\\G' > #{@master_status_file}"
-      system(master_status_cmd)
-    end
-
-    def flush_tables_with_read_lock
-      pipe = IO.popen("mysql -u #{@username} -p#{@password}", 'w')
-      @read_lock_pid = pipe.pid
-
-      pipe.puts('flush tables with read lock;')
-      sleep(@lock_wait_timeout)
-
-      if (thread_id = waiting_read_lock_thread) != ''
-        Process.kill('TERM', @read_lock_pid)
-
-        # after killing the process the mysql thread is still hanging out, need to kill it directly
-        kill_thread_cmd = "mysql -u #{@username} -p#{@password} -e'kill #{thread_id};'"
-        system(kill_thread_cmd)
-        abort "Read lock not acquired after #{@lock_wait_timeout} second timeout.  Killed request and aborting backup."
-      end
-
-      true
-    end
-
-    def unlock_tables
-      # technically we don't actually have to do anything here since the spawned
-      # process that has the read lock will die with this one but it doesn't hurt
-      # to be safe
-      Process.kill('TERM', @read_lock_pid)
-      true
-    end
 
-    def disconnect
-      @dbh.disconnect
+    def say(msg, newline = true)
+      return if @opts[:quiet]
+      print("#{msg}#{"\n" if newline}")
     end
   end
 end