ey_cloud_server 1.4.60 → 1.4.61

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjkxYjFlMDg5ODQwOGNmMmVjZmQyM2VlNDhlNzQ3ZjUwNmY3MzJiYQ==
+    ODliZWU4ZDUzNjk1NTExYzdiMTJiMjE3MzUxOGI2Y2VjYmUwNzg4ZQ==
   data.tar.gz: !binary |-
-    ZWEzZmJhMjQ5Y2I2YmFkZWQ1NDVkYWM1OTM5YmIyYzEyNDhmMmI0Zg==
+    OWI2Nzg4YWI5ZDFhNGZiNDcwOGM3ZGFmYTE5OTE4NzM2YzMyOGY3Zg==
 SHA512:
   metadata.gz: !binary |-
-    OWI2N2I3ZWJjNjlhN2EwZDBlZjlhMWJkZjRjYzc1MWZhNGVkZDViZDVjYWEz
-    NWUxMGI3NzcyM2EyNmVmZDAzYzQ1MGVkZmY2ZDQyNTBlMGYwZjM3OThmMDE0
-    YWU5ODA3MDc2NTc4NDYxZTQzYjU3Mjg1MTU4YTM3ZTkwNzNiOWI=
+    YzQxNzY3OTE3YWViZTZlMmFkNTc5ZWJiMWZiYTYyN2FmMDNkYTQyYjk5ZWQ4
+    MGVhOGRmNzVjZDVkMGFjYzFkN2UxMjdlODk3MDczNDVhMjI1MjFjOTg2YmUy
+    ZDRmY2I3YmU2OTk3N2E1MmU2ODc4NjBiMTQzNWNjNzcyYjNhYjI=
   data.tar.gz: !binary |-
-    ZTBjZjEzNTUzZGI5NTVlNjc1ZWYwZTliYjNhOWYyNjIyYzkyNzc4MjA4YWI2
-    MjJiZDg3ZTI5MTEzZTNlZGUzYTViNTc3Mjc1NDYyYjNlNzAwMjBlMDg5ZGYx
-    ZGY4ZTEwNjM5NTc2NjZlMTk5NWYwNDUzOWYwNWFmY2VlMjNjZWY=
+    YTRlZjdjYmE2ODk5NTQ0YzE1ODkwOTAwYWM1OWZhMDRiMDhhZmYxZWYxZmQ3
+    ZTkwZWY0ZGRkZTE2MDZlNjU4ZGNlZWRmMzBjNTVmMGUyZmNlOWE2NDZjZjll
+    NDZmYTI4N2ZjZTdjNWE5MDA4NjAwMGE4NjUwODdlODZiODgzMzc=

data/features/step_definitions/mysql.rb

@@ -1,10 +1,12 @@
 Given \
     /^there is a mysql database \(([^\)]*)\)$/ do |db_key|
   create_mysql_database(db_key)
+  setup_dna({:db_stack_name => "mysql5_5"})
 end
 
 Given \
     /^the "([^\"]*)" mysql database is dropped$/ do |db_key|
   db_name = created_mysql_dbs[db_key]
   drop_mysql_database(db_name)
+  teardown_dna
 end

data/lib/ey_backup.rb

@@ -147,7 +147,7 @@ module EY
         if ! @options.key?(:dbhost) or @options[:dbhost] == nil or @options[:dbhost] == ""
           @options[:dbhost] = 'localhost'
         end
-        @engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id], @options[:force], @options[:allow_concurrent], @options[:skip_analyze])
+        @engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id], @options[:force], @options[:allow_concurrent], @options[:skip_analyze], @options[:log_coordinates])
       end
 
       def dispatch

data/lib/ey_backup/cli.rb

@@ -9,18 +9,50 @@ module EY
 
         config_path = options[:config] || "/etc/.#{options[:engine]}.backups.yml"
 
-        config_for(config_path).merge(options)
+        options = config_for(config_path).merge(options)
+        options = late_options.merge(options)
+        
+        options[:tmp_dir] = '/mnt/tmp' if options[:tmp_dir].nil?
+        options = delayed_engine(File.join(options[:tmp_dir], '/chef/dna.json')).merge(options){|k, v1, v2| v2.nil? ? v1 : v2}
+        abort("Unable to determine database stack from dna; specify the engine with -e") if options[:engine].nil?
+        options
       end
 
       def default_options
         {
           :command => :new_backup,
           :format => "gzip",
-          :engine => 'mysql',
+          :engine => db_stack_name('/etc/chef/dna.json'),
           :verbose => false,
         }
       end
       
+      def late_options
+        {
+          :log_coordinates => false,
+          :skip_analyze => false,
+          :allow_concurrent => false,
+        }
+      end
+      
+      def delayed_engine(path)
+        {
+          :engine => db_stack_name(path),
+        }
+      end
+      
+      def db_stack_name(file)
+
+        if File.exist?(file)
+          case %x{cat #{file} | grep db_stack_name}
+          when /mysql/
+            'mysql'
+          when /postgres/
+            'postgresql'
+          end
+        end
+      end
+      
       def verify_restore
         res = ''
         puts "This will overwrite your database, are you sure you would like to proceed (Y/n)?"
@@ -118,12 +150,14 @@ module EY
             options[:force] = true
           end
           
-          options[:allow_concurrent] = false
+          opts.on("--log_coordinates", "Record MySQL binary log position so backup can be used with replication or Point in Time functions.") do
+            options[:log_coordinates] = true
+          end
+          
           opts.on("--allow_concurrent", "Allow eybackup process to run concurrently with other eybackup processes.") do
             options[:allow_concurrent] = true
           end
           
-          options[:skip_analyze] = false
           opts.on("--skip_analyze", "Skip automatic analyze during PostgreSQL Restore operations.") do
             options[:skip_analyze] = true
           end

data/lib/ey_backup/engine.rb

@@ -3,7 +3,7 @@ module EY
     class Engine < Base
       include Spawner
 
-      attr_reader :username, :password, :host, :key_id, :force, :allow_concurrent, :skip_analyze
+      attr_reader :username, :password, :host, :key_id, :force, :allow_concurrent, :skip_analyze, :log_coordinates
 
       def self.label
         @label
@@ -26,8 +26,8 @@ module EY
           EY::Backup.logger.fatal("Unknown database engine: #{label}")
       end
 
-      def initialize(username, password, host, key_id, force, allow_concurrent, skip_analyze)
-        @username, @password, @host, @key_id, @force, @allow_concurrent, @skip_analyze  = username, password, host, key_id, force, allow_concurrent, skip_analyze
+      def initialize(username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates)
+        @username, @password, @host, @key_id, @force, @allow_concurrent, @skip_analyze, @log_coordinates = username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates
       end
       
       def block_concurrent(db = nil)

data/lib/ey_backup/engines/mysql_engine.rb

@@ -57,14 +57,14 @@ module EY
       end
       
       def server_id
-        if log_bin_on?
+        if log_bin_on? and log_coordinates
           stdout = %x{mysql #{username_option} #{host_option} -BN -e"select @@global.server_id"}
           "echo '-- Server_id: #{stdout.strip}' && "
         end 
       end
       
       def master_data_option
-        "--master-data=2" if log_bin_on?
+        "--master-data=2" if log_bin_on? and log_coordinates
       end
       
       def databases_option(db)

data/lib/ey_backup/engines/postgresql_engine.rb

@@ -6,7 +6,7 @@ module EY
       def dump(database_name, basename)
         file = basename + '.dump'
 
-        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --create --format=c -Upostgres #{database_name} "
+        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --create --format=c -U#{username} #{database_name} "
 
         if gpg?
           command << " | " << GPGEncryptor.command_for(key_id)
@@ -25,26 +25,32 @@ module EY
         if file =~ /.gpz$/ # GPG?
           abort "\nCannot restore a GPG backup directly; decrypt the file (#{file}) using your key and then load with pg_restore.
   To decrypt a backup: https://support.cloud.engineyard.com/hc/en-us/articles/205413948-Use-PGP-Encrypted-Database-Backups-with-Engine-Yard-Cloud#restore
-  Once decrypted, restore with: `pg_restore -h #{host} --format=c --clean -Upostgres -d #{database_name} <filename>`\n\n"
+  Once decrypted, restore with: `pg_restore -h #{host} --format=c --clean -U#{username} -d #{database_name} <filename>`\n\n"
         end
 
         cycle_database(database_name)
+        
+        # Exclude Extension Comments
+        toc_file = "#{file}.toc"
+        table_of_contents(file, toc_file)
 
         command = "cat #{file}"
 
-        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c -Upostgres -d #{database_name}"
+        command << " | PGPASSWORD='#{password}' pg_restore -L #{toc_file} -h #{host} --format=c -U#{username} -d #{database_name}"
 
         run(command, database_name)
         
+        system("rm #{toc_file}") if File.exists?(toc_file)
+        
         # Analyze database unless disabled
         unless skip_analyze
           verbose "Analyzing database '#{database_name}', use --skip-analyze to skip this step."
-          %x{PGPASSWORD='#{password}' vacuumdb -h #{host} -Upostgres -d #{database_name} --analyze-only}
+          %x{PGPASSWORD='#{password}' vacuumdb -h #{host} -U#{username} -d #{database_name} --analyze-only}
         end
       end
 
       def check_connections(database_name)
-        active_connections = %x{PGPASSWORD='#{password}' psql -U postgres -h #{host} -t -c "select count(*) from pg_stat_activity where datname='#{database_name}';"}
+        active_connections = %x{PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c "select count(*) from pg_stat_activity where datname='#{database_name}';"}
         
         if active_connections.to_i > 0
           res = ''
@@ -64,13 +70,13 @@ module EY
       end
       
       def cancel_connections(database_name)
-        %x{psql -U postgres -h #{host} -c"SELECT pg_terminate_backend(pg_stat_activity.pid)
+        %x{psql -U#{username} -h #{host} postgres -c"SELECT pg_terminate_backend(pg_stat_activity.pid)
           FROM pg_stat_activity
           WHERE pg_stat_activity.datname = '#{database_name}';"}
       end
       
       def drop_database(database_name)
-        command = "PGPASSWORD='#{password}' dropdb -h #{host} -Upostgres #{database_name}"
+        command = "PGPASSWORD='#{password}' dropdb -h #{host} -U#{username} #{database_name}"
         verbose "Dropping Database with: #{command}"
         %x{#{command}}
       end
@@ -82,14 +88,23 @@ module EY
       end
       
       def check_if_replica
-        stdout = %x{PGPASSWORD='#{password}' psql -U postgres -h #{host} -t -c "select pg_is_in_recovery()"| head -n 1}
+        command = "PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t -c 'select pg_is_in_recovery()' postgres| head -n 1"
+        verbose "Checking for replica state with: #{command}"
+        stdout = %x{#{command}}
         unless stdout.chomp =~ /^\W*f$/
           EY::Backup.logger.fatal(%Q{ERROR: Target host: '#{host}' is currently a replica in recovery mode; restore operations need to be processed against the master.})
         end
       end
       
+      def table_of_contents(file, toc_file)
+        command = %Q{pg_restore -l #{file} | sed -e 's/^\\\(.* COMMENT - EXTENSION .*\\\)/;\\1/g' \
+          -e 's/^\\\(.* rdsadmin$\\\)/;\\1/g' > #{toc_file}}
+        verbose "Creating table of contents: #{command}"
+        %x{#{command}}
+      end
+      
       def create_command(database_name)
-        %x{PGPASSWORD='#{password}' psql -U postgres -h #{host} -t -c "SELECT 'CREATE DATABASE ' || datname ||
+        command="PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c \"SELECT 'CREATE DATABASE ' || datname ||
           ' WITH OWNER ' || pg_user.usename ||
           CASE (select pg_encoding_to_char(encoding) from pg_database where datname='template1') 
           WHEN pg_encoding_to_char(encoding) 
@@ -103,8 +118,9 @@ module EY
         FROM pg_database
         INNER JOIN pg_user
           ON pg_user.usesysid = pg_database.datdba
-        WHERE datname = '#{database_name}'"
-        }
+        WHERE datname = '#{database_name}'\""
+        verbose "Getting create info: #{command}"
+        %x{#{command}}
       end
       
       def cycle_database(database_name)
@@ -114,7 +130,7 @@ module EY
         else
           check_connections(database_name)
           drop_database(database_name)
-          %x{PGPASSWORD='#{password}' psql -U postgres -h #{host} -t -c "#{create_cmd}"}
+          %x{PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c "#{create_cmd}"}
         end
       end
 

data/lib/ey_cloud_server/version.rb

@@ -1,5 +1,5 @@
 module EY
   module CloudServer
-    VERSION = '1.4.60'
+    VERSION = '1.4.61'
   end
 end

data/spec/ey_backup/backup_spec.rb

@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
 describe EY::Backup do
   before(:each) do
     @db_name = create_mysql_database('first')
+    setup_dna({:db_stack_name => "mysql5_5"})
   end
   
   after(:each) do
     drop_mysql_database(@db_name)
+    teardown_dna
   end
 
   describe "#list" do

data/spec/ey_backup/cli_spec.rb

@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
 describe "MySQL Backups" do
   before(:each) do
     @db_name = create_mysql_database('first')
+    setup_dna({:db_stack_name => "mysql5_5"})
   end
   
   after(:each) do
     drop_mysql_database(@db_name)
+    teardown_dna
   end
 
   describe "--quiet" do

data/spec/ey_backup/mysql_backups_spec.rb

@@ -3,11 +3,13 @@ require File.dirname(__FILE__) + '/spec_helper'
 describe "MySQL Backups" do
   before(:each) do
     @db_name = create_mysql_database('first')
+    setup_dna({:db_stack_name => "mysql5_5"})
     mk_tmp
   end
   
   after(:each) do
     drop_mysql_database(@db_name)
+    teardown_dna
   end
 
   it "makes a backup" do
@@ -148,12 +150,14 @@ end
 describe "MySQL Backups" do
   before(:each) do
     @dbs = [create_mysql_database('first'), create_mysql_database('second')]
+    setup_dna({:db_stack_name => "mysql5_5"})
   end
   
   after(:each) do
     @dbs.each do |db|
       drop_mysql_database(db)
     end
+    teardown_dna
   end
 
   it "makes a backup" do
@@ -172,10 +176,12 @@ describe "a multi-region backup" do
   describe "in japan!" do
     before(:each) do
       @db_name = create_mysql_database('speaking_japanese', 'ap-northeast-1')
+      setup_dna({:db_stack_name => "mysql5_5"})
     end
     
     after(:each) do
       drop_mysql_database(@db_name)
+      teardown_dna
     end
 
     it "makes a split backup" do
@@ -211,10 +217,12 @@ describe "a multi-region backup" do
   describe "in europe!" do
     before(:each) do
       @db_name = create_mysql_database('speaking_esperanto', 'eu-west-1')
+      setup_dna({:db_stack_name => "mysql5_5"})
     end
     
     after(:each) do
       drop_mysql_database(@db_name)
+      teardown_dna
     end
 
     it "makes a split backup" do

data/spec/ey_backup/postgres_backups_spec.rb

@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
 describe "Postgres Backups" do
   before(:each) do
     @db_name = create_postgresql_database('first')
+        setup_dna({:db_stack_name => "postgres9_5"})
   end
   
   after(:each) do
     drop_postgresql_database(@db_name)
+    teardown_dna
   end
 
   it "makes a custom format backup" do
@@ -18,6 +20,16 @@ describe "Postgres Backups" do
 
     stdout.should match(/0:#{@db_name}.*\.dump$/)
   end
+  
+  it "makes a custom backup and detects that the db is Postgres" do
+    EY::Backup.run(["-c", backup_config_file])
+
+    reset_logger
+
+    EY::Backup.run(["-c", backup_config_file, "-l", @db_name, "-e", "postgresql"])
+
+    stdout.should match(/0:#{@db_name}.*\.dump$/)
+  end
 
   it "makes a split backup" do
     EY::Backup.run(["-c", backup_config_file, "-e", "postgresql", "-s", "100"])

data/spec/helpers.rb

@@ -73,16 +73,17 @@ module Helpers
     end
   end
 
-  def teardown_dna(data)
-    FileUtils.rm_f("#{tmp_dir}/dna.json")
+  def teardown_dna()
+    FileUtils.rm_f("#{tmp_dir}/chef/dna.json")
+    FileUtils.rm_r("#{tmp_dir}/chef")
   end
 
   def setup_dna(data)
-    FileUtils.mkdir_p("#{tmp_dir}")
-    if File.exist?("#{tmp_dir}/dna.json")
-      FileUtils.rm("#{tmp_dir}/dna.json")
+    FileUtils.mkdir_p("#{tmp_dir}/chef")
+    if File.exist?("#{tmp_dir}/chef/dna.json")
+      FileUtils.rm("#{tmp_dir}/chef/dna.json")
     end
-    File.open("#{tmp_dir}/dna.json", "w") do |f|
+    File.open("#{tmp_dir}/chef/dna.json", "w") do |f|
       f.puts data.to_json
     end
   end

data/spec/log/ey_flex_postgresql_db_thecae.sizes

@@ -0,0 +1 @@
+2017-08-29 15:44:07 -0400 1.7 KB

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ey_cloud_server
 version: !ruby/object:Gem::Version
-  version: 1.4.60
+  version: 1.4.61
 platform: ruby
 authors:
 - EngineYard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-17 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -279,7 +279,7 @@ files:
 - spec/gpg.public
 - spec/gpg.sekrit
 - spec/helpers.rb
-- spec/log/ey_flex_postgresql_db_wanting.sizes
+- spec/log/ey_flex_postgresql_db_thecae.sizes
 - spec/snapshot_minder_spec.rb
 - spec/spec_helper.rb
 homepage: http://developer.engineyard.com
@@ -320,7 +320,7 @@ test_files:
 - spec/gpg.public
 - spec/gpg.sekrit
 - spec/helpers.rb
-- spec/log/ey_flex_postgresql_db_wanting.sizes
+- spec/log/ey_flex_postgresql_db_thecae.sizes
 - spec/snapshot_minder_spec.rb
 - spec/spec_helper.rb
 - features/downloading_a_mysql_backup.feature

data/spec/log/ey_flex_postgresql_db_wanting.sizes

@@ -1 +0,0 @@
-2017-05-17 02:10:11 -0400 1.7 KB