ey_cloud_server 1.3.1 → 1.4.5.pre

data/lib/ey_backup/spawner.rb

@@ -0,0 +1,54 @@
+module EY
+  module Backup
+    module Spawner
+      extend Forwardable
+      extend self
+
+      CHUNK_SIZE = 4096
+
+      def_delegator EY::Backup, :logger
+
+      def spawn(command, stdout = nil, stdin = nil, stderr = logger.stderr)
+        ioify(stdout, stdin, stderr) do |o, i, e|
+          ios = {:stderr => e}
+          ios[:stdout] = o if o
+          ios[:stdin] = i if i
+          result = Open4.spawn([command], ios)
+
+        end
+      end
+
+      def run(command)
+        pid, *_ = Open4.popen4(command)
+        Process::waitpid2(pid)
+      end
+
+      def popen(command, stdout, stdin = nil, stderr = logger.stderr)
+        ioify(stdout, stdin, stderr) do |o, i, e|
+          Open4.popen4(command) do |pid, stdin, stdout, stderr|
+            while chunk = i.read(CHUNK_SIZE)
+              stdin << chunk
+            end
+
+            stdin.flush
+            stdin.close
+          end
+        end
+      end
+
+      def ioify(stdout, stdin, stderr, &block)
+        if String === stdin
+          File.open(stdin, 'r') do |f|
+            ioify(stdout, f, stderr, &block)
+          end
+        elsif String === stdout
+          File.open(stdout, 'w') do |f|
+            ioify(f, stdin, stderr, &block)
+          end
+        else
+          yield stdout, stdin, stderr
+        end
+      end
+    end
+  end
+end

data/lib/ey_cloud_server/mysql_start.rb

@@ -6,12 +6,13 @@ module EY
         new(options).run
       end
 
-      attr_accessor :password, :check, :timeout
+      attr_accessor :password, :check, :start_timeout, :stop_timeout
 
       def initialize(options = {})
-        @password = options[:password]  || abort("ERROR: You must provide a password.")
-        @check    = options[:check]     || abort("ERROR: You must a check interval.")
-        @timout   = options[:timeout]   || abort("ERROR: You must provide a timeout.")
+        @password      = options[:password]      || abort("ERROR: You must provide a password.")
+        @check         = options[:check]         || abort("ERROR: You must a check interval.")
+        @start_timeout = options[:start_timeout] || abort("ERROR: You must provide a start timeout.")
+        @stop_timeout  = options[:stop_timeout]  || 1200
 
         abort("ERROR: You must be root.") unless Process.euid == 0
       end
@@ -26,19 +27,17 @@ module EY
             # test to make sure mysql is running
             if mysql_running()
               log_mysql_event("MySQL did start and is working through crash recovery")
-              sleeptime = check
               slept = 0
-              sleeplimit = timeout
               loop {
                 if system("mysqladmin -p#{password} status") # mysql is accessible
                   log_mysql_event("MySQL completed crash recovery, performing clean restart")
                   system('killall -TERM mysqld')          # safe shutdown of mysql
                   termslept = 0
-                  termlimit = 120
+                  termlimit = stop_timeout
                   loop {
                     break if not mysql_running()
-                    sleep(sleeptime)
-                    termslept = termslept + sleeptime
+                    sleep(check)
+                    termslept = termslept + check
                     if termslept > termlimit
                       log_mysql_event("MySQL did not shut down cleanly within the time limit, killing")
                       system('killall -9 mysqld')
@@ -50,11 +49,11 @@ module EY
                   end
                 else
                   log_mysql_event("MySQL has been starting for #{slept/60} minutes and is still attempting to start") if slept % 120 == 0
-                  sleep(sleeptime)
+                  sleep(check)
                 end
               break if not mysql_running()
-              slept = slept + sleeptime
-              if slept > sleeplimit
+              slept = slept + check
+              if slept > start_timeout
                 log_mysql_event("MySQL was not able to start after #{slept/60} minutes and was forcibly killed")
                 system("killall -9 mysqld")
               end
@@ -134,8 +133,16 @@ module EY
             options[:check] = check.to_i
           end
 
-          opts.on("-t", "--timeout NUMBER", "Maximum wait time in seconds.") do |timeout|
-            options[:timeout] = timeout.to_i
+          opts.on("-t", "--timeout NUMBER", "Maximum wait time in seconds. (DEPRECATED)") do |start_timeout|
+            options[:start_timeout] = start_timeout.to_i
+          end
+
+          opts.on("-a", "--start NUMBER", "Startup timeout in seconds.") do |start_timeout|
+            options[:start_timeout] = start_timeout.to_i
+          end
+
+          opts.on("-o", "--stop NUMBER", "Graceful termination timeout in seconds.") do |stop_timeout|
+            options[:stop_timeout] = stop_timeout.to_i
           end
         end
 

metadata

@@ -1,128 +1,154 @@
 --- !ruby/object:Gem::Specification 
 name: ey_cloud_server
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 961916000
+  prerelease: true
   segments: 
   - 1
-  - 3
-  - 1
-  version: 1.3.1
+  - 4
+  - 5
+  - pre
+  version: 1.4.5.pre
 platform: ruby
-authors: 
-- Engine Yard Engineers
+authors: []
+
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2010-05-06 00:00:00 -07:00
+date: 2010-10-05 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: json
-  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
+  name: json
+  prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: right_aws
-  prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
+  name: right_aws
+  prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: open4
-  prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
+        hash: 55
         segments: 
         - 0
         - 9
         - 6
         version: 0.9.6
   type: :runtime
+  name: open4
+  prerelease: false
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: aws-s3
-  prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
+  name: aws-s3
+  prerelease: false
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: rest-client
-  prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
   type: :runtime
+  name: rest-client
+  prerelease: false
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: ey_stonith
-  prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
-        - 3
-        - 0
-        version: 0.3.0
+        version: "0"
   type: :runtime
+  name: mysql
+  prerelease: false
   version_requirements: *id006
-description: "Server side components for Engine Yard's cloud "
+description: Server side components for Engine Yard's cloud
 email: 
-- ninja@engineyard.com
 executables: 
 - eybackup
 - ey-snapshots
 - ey-agent
 - mysql_start
+- binary_log_purge
+- clear_binlogs_from_slave
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
-- lib/ey-flex/backups.rb
 - lib/ey-flex/big-brother.rb
 - lib/ey-flex/bucket_minder.rb
-- lib/ey-flex/ey-api.rb
-- lib/ey-flex/mysql_database.rb
-- lib/ey-flex/postgresql_database.rb
-- lib/ey-flex/snapshot_minder.rb
 - lib/ey-flex/version.rb
-- lib/ey-flex.rb
+- lib/ey-flex/snapshot_minder.rb
+- lib/ey-flex/ey-api.rb
+- lib/ey_backup.rb
 - lib/ey_cloud_server/mysql_start.rb
 - lib/ey_cloud_server.rb
-- bin/eybackup
-- bin/ey-snapshots
+- lib/ey-flex.rb
+- lib/ey_backup/processors/splitter.rb
+- lib/ey_backup/processors/gzipper.rb
+- lib/ey_backup/processors/gpg_encryptor.rb
+- lib/ey_backup/engines/postgresql_engine.rb
+- lib/ey_backup/engines/mysql_engine.rb
+- lib/ey_backup/backup_set.rb
+- lib/ey_backup/loader.rb
+- lib/ey_backup/engine.rb
+- lib/ey_backup/logger.rb
+- lib/ey_backup/cli.rb
+- lib/ey_backup/dumper.rb
+- lib/ey_backup/spawner.rb
+- lib/ey_backup/base.rb
+- bin/clear_binlogs_from_slave
+- bin/binary_log_purge
 - bin/ey-agent
 - bin/mysql_start
+- bin/ey-snapshots
+- bin/eybackup
 has_rdoc: true
-homepage: http://github.com/engineyard
+homepage: 
 licenses: []
 
 post_install_message: 
@@ -131,23 +157,29 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
-rubyforge_project: http://gemcutter.org
-rubygems_version: 1.3.6
+rubyforge_project: 
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Server side components for Engine Yard's cloud

data/lib/ey-flex/backups.rb

@@ -1,233 +0,0 @@
-module AWS::S3
-  class S3Object
-    def <=>(other)
-      DateTime.parse(self.about['last-modified']) <=> DateTime.parse(other.about['last-modified'])
-    end
-  end
-end
-
-module EY::Flex
-  class DatabaseEngine
-    def self.register_as(name)
-      EY::Flex::Backups::ENGINES[name] = self
-    end
-
-    def initialize(backups)
-      @backups = backups
-    end
-
-    def dump_database(name)
-      raise "Implement #dump_database in #{self.class}"
-    end
-
-    def dbuser
-      @backups.config[:dbuser]
-    end
-
-    def dbpass
-      @backups.config[:dbpass]
-    end
-  end
-
-  class Backups
-    class BackupNotFound < EY::Flex::Error; end
-
-    def self.run(args)
-      options = {:command => :new_backup}
-
-      # Build a parser for the command line arguments
-      opts = OptionParser.new do |opts|
-        opts.version = "0.0.1"
-
-        opts.banner = "Usage: eybackup [-flag] [argument]"
-        opts.define_head "eybackup: backing up your shit since way back when..."
-        opts.separator '*'*80
-        
-        opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
-          options[:db] = (db || 'all')
-          options[:command] = :list
-        end
-        
-        opts.on("-n", "--new-backup", "Create new mysql backup") do
-          options[:command] = :new_backup
-        end
-        
-        opts.on("-c", "--config CONFIG", "Use config file.") do |config|
-          options[:config] = config
-        end
-        
-        opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index|
-          options[:command] = :download
-          options[:index] = index
-        end
-
-        opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
-          options[:engine] = engine
-        end
-
-        opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index|
-          options[:command] = :restore
-          options[:index] = index
-        end
-
-        opts.on("-q", "--quiet", "Supress output to STDOUT") do
-          options[:quiet] = true
-        end
-
-      end
-
-      opts.parse!(args)
-
-      options[:engine] ||= 'mysql'
-      options[:config] ||= "/etc/.#{options[:engine]}.backups.yml"
-
-      eyb = new(options)
-
-      case options[:command]
-      when :list
-        eyb.list options[:db]
-      when :new_backup
-        eyb.new_backup
-      when :download
-        eyb.download(options[:index])
-      when :restore
-        eyb.restore(options[:index])
-      end
-      eyb.cleanup
-    rescue EY::Flex::Error => e
-      abort e.message
-    end
-
-    ENGINES = {}
-
-    def initialize(options = {})
-      @quiet = options[:quiet]
-      engine_klass = ENGINES[options[:engine]] || raise("Invalid database engine: #{options[:engine].inspect}")
-      @engine = engine_klass.new(self)
-
-      load_config(options[:config])
-
-      AWS::S3::Base.establish_connection!(
-          :access_key_id     => config[:aws_secret_id],
-          :secret_access_key => config[:aws_secret_key]
-        )
-      @databases = config[:databases]
-      @keep = config[:keep]
-      @bucket = "ey-backup-#{Digest::SHA1.hexdigest(config[:aws_secret_id])[0..11]}"
-      @tmpname = "#{Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')}.sql.gz"
-      @env = config[:env]
-      FileUtils.mkdir_p '/mnt/backups'
-      FileUtils.mkdir_p '/mnt/tmp'
-      begin
-        AWS::S3::Bucket.find(@bucket)
-      rescue AWS::S3::NoSuchBucket
-        AWS::S3::Bucket.create(@bucket)
-      end
-      
-      FileUtils.mkdir_p self.backup_dir
-    end
-    attr_reader :config
-
-    def load_config(filename)
-      if File.exist?(filename)
-        @config = YAML::load(File.read(filename))
-      else
-        abort "You need to have a backup file at #{filename}"
-      end
-    end
-
-    def new_backup
-      @databases.each do |db|
-        backup_database(db)
-      end  
-    end
-    
-    def say(msg, newline = true)
-      return if @quiet
-      print("#{msg}#{"\n" if newline}")
-    end
-
-    def backup_database(database)
-      File.open("#{self.backup_dir}/#{database}.#{@tmpname}", "w") do |f|
-        say "doing database: #{database}"
-        @engine.dump_database(database, f)
-      end
-
-      File.open("#{self.backup_dir}/#{database}.#{@tmpname}") do |f|
-        path = "#{@env}.#{database}/#{database}.#{@tmpname}"
-        AWS::S3::S3Object.store(path, f, @bucket, :access => :private)
-        say "successful backup: #{database}.#{@tmpname}"
-      end
-    end
-
-    def download(index)
-      idx, db = index.split(":")
-      raise Error, "You didn't specify a database name: e.g. 1:rails_production" unless db
-
-      if obj = list(db)[idx.to_i]
-        filename = normalize_name(obj)
-        say "downloading: #{filename}"
-        File.open(filename, 'wb') do |f|
-          say ".", false
-          obj.value {|chunk| f.write chunk }
-        end
-        say "finished"
-        [db, filename]
-      else
-        raise BackupNotFound, "No backup found for database #{db.inspect}: requested index: #{idx}"
-      end
-    end
-    
-    def restore(index)
-      db, filename = download(index)
-      File.open(filename) do |f|
-        @engine.restore_database(db, f)
-      end
-    end
-    
-    def cleanup
-      begin
-        list('all')[0...-(@keep*@databases.size)].each do |o| 
-          say "deleting: #{o.key}"
-          o.delete
-        end
-      rescue AWS::S3::S3Exception, AWS::S3::Error
-        nil # see bucket_minder cleanup note regarding S3 consistency
-      end
-    end
-    
-    def normalize_name(obj)
-      obj.key.gsub(/^.*?\//, '')
-    end
-    
-    def find_obj(name)
-      AWS::S3::S3Object.find name, @bucket
-    end
-    
-    def list(database='all')
-      say "Listing database backups for #{database}"
-      backups = []
-      if database == 'all'
-        @databases.each do |db|
-          backups << AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{db}")
-        end
-        backups = backups.flatten.sort
-      else
-        backups = AWS::S3::Bucket.objects(@bucket, :prefix => "#{@env}.#{database}").sort
-      end
-
-      say "#{backups.size} backup(s) found"
-
-      backups.each_with_index do |b,i|
-        say "#{i}:#{database} #{normalize_name(b)}"
-      end
-      backups
-    end
-    
-    protected
-    def backup_dir
-      "/mnt/tmp"
-    end
-    
-  end
-end