ey_cloud_server 1.4.47 → 1.4.49

data/spec/spec_helper.rb

@@ -1,10 +1,9 @@
 require File.dirname(__FILE__) + "/../lib/ey-flex"
 
-require 'ruby-debug'
 require 'fakeweb'
 require 'fakefs/safe'
 require 'randexp'
-require 'fog'
+require 'fog/aws'
 
 # Made tests work with actual storage; better tests.  - rpowell Fri Aug  5 17:53:17 PDT 2011
 #

metadata

@@ -1,156 +1,155 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ey_cloud_server
-version: !ruby/object:Gem::Version 
-  version: 1.4.47
+version: !ruby/object:Gem::Version
+  version: 1.4.49
 platform: ruby
-authors: 
+authors:
 - EngineYard
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2015-05-12 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2016-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: json
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.4.4
-    - - <=
-      - !ruby/object:Gem::Version 
-        version: 1.4.6
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: right_http_connection
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        version: 1.2.4
+      - !ruby/object:Gem::Version
+        version: 1.8.3
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: right_aws
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        version: 2.0.0
-  type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+- !ruby/object:Gem::Dependency
   name: open4
-  prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
         version: 1.3.0
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 1.6.6.2
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: aws-s3
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: 0.6.2
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+- !ruby/object:Gem::Dependency
+  name: aws-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.3
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: fog
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: fog-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.30.0
+      - !ruby/object:Gem::Version
+        version: 0.3.0
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: mysql
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: 2.8.1
-  type: :runtime
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
   name: ey_enzyme
-  prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 1.1.7
   type: :runtime
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
-  name: ey_instance_api_client
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: 0.1.1
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.7
+- !ruby/object:Gem::Dependency
+  name: ey_instance_api_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.11
   type: :runtime
-  version_requirements: *id010
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
-    requirements: 
-    - &id012 
-      - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.11
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id011
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Miscellaneous EY server utilities
-email: 
+email:
 - engineering@engineyard.com
-executables: 
-- binary_log_purge
-- clear_binlogs_from_slave
-- ey-agent
+executables:
 - ey-snapshots
 - eybackup
-- mysql_start
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- bin/binary_log_purge
-- bin/clear_binlogs_from_slave
-- bin/ey-agent
+files:
+- LICENSE
+- README.rdoc
+- Rakefile
+- TODO
 - bin/ey-snapshots
 - bin/eybackup
-- bin/mysql_start
-- lib/ey-flex/big-brother.rb
+- features/downloading_a_mysql_backup.feature
+- features/making_a_postgresql_backup.feature
+- features/restoring_postgres_backup.feature
+- features/step_definitions/backups.rb
+- features/step_definitions/commands.rb
+- features/step_definitions/mysql.rb
+- features/step_definitions/postgresql.rb
+- features/support/env.rb
+- lib/ey-flex.rb
 - lib/ey-flex/bucket_minder.rb
 - lib/ey-flex/ey-api.rb
 - lib/ey-flex/snapshot_minder.rb
-- lib/ey-flex.rb
+- lib/ey_backup.rb
 - lib/ey_backup/backend.rb
 - lib/ey_backup/backup_set.rb
 - lib/ey_backup/base.rb
@@ -166,17 +165,11 @@ files:
 - lib/ey_backup/processors/gzipper.rb
 - lib/ey_backup/processors/splitter.rb
 - lib/ey_backup/spawner.rb
-- lib/ey_backup.rb
-- lib/ey_cloud_server/mysql_start.rb
-- lib/ey_cloud_server/version.rb
 - lib/ey_cloud_server.rb
-- README.rdoc
-- Rakefile
-- LICENSE
-- TODO
-- spec/big-brother_spec.rb
+- lib/ey_cloud_server/version.rb
 - spec/bucket_minder_spec.rb
 - spec/config-example.yml
+- spec/config.yml
 - spec/ey_api_spec.rb
 - spec/ey_backup/backend_spec.rb
 - spec/ey_backup/backup_spec.rb
@@ -190,41 +183,33 @@ files:
 - spec/helpers.rb
 - spec/snapshot_minder_spec.rb
 - spec/spec_helper.rb
-- features/downloading_a_mysql_backup.feature
-- features/making_a_postgresql_backup.feature
-- features/restoring_postgres_backup.feature
-- features/step_definitions/backups.rb
-- features/step_definitions/commands.rb
-- features/step_definitions/mysql.rb
-- features/step_definitions/postgresql.rb
-- features/support/env.rb
 homepage: http://developer.engineyard.com
 licenses: []
-
 metadata: {}
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - *id012
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - *id012
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Server side components for Engine Yard's cloud
-test_files: 
-- spec/big-brother_spec.rb
+test_files:
 - spec/bucket_minder_spec.rb
 - spec/config-example.yml
+- spec/config.yml
 - spec/ey_api_spec.rb
 - spec/ey_backup/backend_spec.rb
 - spec/ey_backup/backup_spec.rb

data/bin/binary_log_purge

@@ -1,263 +0,0 @@
-#!/usr/bin/env ruby
-
-# Filename: binary_log_purge.rb
-# Author: Tyler Poland
-# Version: 0.2
-# Purpose: Script to check current state of all replica databases and
-# 	purge binary logs from the master based on the position of any
-# 	and all replica databases.
-
-# Changelog 0.1 -> 0.2
-# - modified binlog check routine to lookup binary log storage in configuration file instead of relying on
-#   binary logs being stored in the data directory
-
-# Changelog 0.2 -> 0.3
-# - Added ability to purge binary logs for standalone master databases
-
-# Changelog 0.3 -> 0.4
-# - Added support for remote tunneled slave with reverse connection on port 13306
-
-# Changelog 0.4 -> 1.0 
-# - Add automatic create user for master to connect to replica
-
-# Changelog 1.0 -> 1.1
-# - Remove unecessary require of mysql
-# - Remove -N from mysql command line (drops header row) for compatibility with 5.1 and 5.5
-
-require 'rubygems'
-require 'net/smtp'
-require 'yaml'
-require 'open3'
-require 'getoptlong'
-
-# Set up logging functions based on desired verbosity level
-def log_info(message)   # may get redefined below
-  puts message
-end
-
-def log_error(message)
-  STDERR.write(message + "\n")
-end
-
-opts = GetoptLong.new(['--quiet', '-q', GetoptLong::NO_ARGUMENT])
-opts.each do |opt, arg|
-  if opt == '--quiet'
-    def log_info(_) end
-  end
-end
-
-
-log_info Time.now
-# Conditional require for JSON if dna file exists
-chef_file = '/etc/chef/dna.json'
-if File.exists?(chef_file)
-  require 'rubygems'
-  require 'json'
-end
-
-# Modify default purge count
-keep_logs = 5
-binpurge_config = '/etc/engineyard/binlogpurge.yml'
-if File.exists?(binpurge_config)
-  options = YAML::load(File.read(binpurge_config))
-  if options['keep'] > 0
-    keep_logs = options['keep'] 
-    log_info "Overriding keep logs from configuration file"
-  end
-end
-
-# function to send error emails
-def failure_message(message)
-  sender = "Database Team <db@engineyard.com"
-  recipients = "db@engineyard.com"
-  hostname = `hostname`.chomp
-  subject = "An error has occurred while purging binary logs on #{hostname}"
-    mailtext = <<EOF
-From: #{sender}
-To: #{recipients}
-Subject: #{subject}
-#{message}
-EOF
-	
-  begin Net::SMTP.start('mail') do |smtp|
-    smtp.sendmail(mailtext, 'root@' + hostname, recipients)
-  end
-  rescue Exception => e
-    log_error "Exception occurred: " + e
-  end
-  
-  exit(1)
-end
-
-# function to retrieve password from .mytop file
-def get_password
-  dbpass = %x{cat /root/.mytop |grep pass |awk -F= '{print $2}'}.chomp
-  failure_message() if dbpass.length < 1
-  dbpass
-end
-
-# function to run query against database
-def run_query(host, user, password, query)
-  options = ''
-  if host == '127.0.0.1'
-    options = options + ' -P13306'
-  end
-  if query == 'show processlist'
-    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -N -e\"#{query}\"|grep 'Binlog'")
-  else
-    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} #{options} -h#{host} -e\"#{query}\"")
-  end
-  query_error = stderr.read
-  if query_error.length > 0
-    log_error "Error caught: #{query_error}"
-    test_add_privilege(user, password, query_error)
-    exit 0
-  end
-  result = stdout.read
-  stdin.close
-  stdout.close
-  stderr.close
-  result
-end
-
-# function to test for user privilege
-def test_add_privilege(user, password, error)
-  full_hostname = %x{hostname --long}.chomp
-  dns_name = %x{hostname -d}.chomp
-  # verify that this is the user privilege error with the root user not having access to the replica
-  if error.match(/ERROR 1045.* Access denied for user 'root'@'.*#{dns_name}' \(using password: YES\)/)
-    # check the master to see if grant based on hostname or IP exists
-    master_ip = %x{hostname -i}.chomp.gsub(/\s+/,'')
-    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} -e\"show grants for 'root'@'#{master_ip}'\"")
-    master_ip_error = stderr.read
-
-    stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} -e\"show grants for 'root'@'#{full_hostname}'\"")
-    full_hostname_error = stderr.read
-    
-    regex = 'ERROR 1141.*There is no such grant defined'
-    if master_ip_error.match(/#{regex}/) || full_hostname_error.match(/#{regex}/)
-      # neither grant is defined on the master so go ahead and add it
-      log_info "The user privilege does not exist on the master, the script will now create it."
-      log_info "This privilege must propagate to the replica via replication, the user may not be available for immediate use."
-      stdin, stdout, stderr = Open3.popen3("mysql -u#{user} -p#{password} -e\"grant all privileges on *.* to 'root'@'#{master_ip}' identified by '#{password}'\"")
-      create_user_error = stderr.read
-      if create_user_error.length > 0
-        log_error "Unable to create user: #{create_user_error}"
-        exit 1
-      end
-    else
-      log_error "The required privilege appears to exist on the master, you may need to wait for replication to process the grant on the Replica"
-      exit 0
-    end
-  end
-end
-
-# function to convert input into yaml
-def yaml_result(file)
-  parse = file.gsub(/^\*.*$/,'').gsub(/^/,' ').gsub(/^\s+/, '  ')
-  yml = YAML.load(parse)
-end
-
-# parse the hostname out of the processlist
-def extract_host(line)
-  line =~ /.+\s+(.+):.+/
-  $1
-end
-
-# function to get replica position from replica host
-def slave_log_file(hostname, user, pass)
-  if hostname == 'localhost'
-    hostname = '127.0.0.1'
-  end
-  
-  q_result = run_query(hostname, user, pass, "show slave status\\G")
-  if q_result.match(/.*Slave_SQL_Running: No.*/)
-    log_error "Slave SQL thread is not running."
-    log_info "The error is: \n#{q_result}"
-    log_error "Unable to continue; exiting!"
-    exit 1
-  end
-   
-  yaml = yaml_result(q_result)
-  yaml["Relay_Master_Log_File"]
-end
-
-dbuser = 'root'
-
-if not dbpassword = get_password
-  failure_message("Password not found for slice, check for /root/.mytop")
-end
-
-mysql_process = %x{ps -ef|grep '[m]ysqld'}.split(/\s+/).select{|item| item.match(/^--/)}
-mysql_params = Hash.new
-mysql_process.each {|i| k, v = i.split('='); mysql_params[k]=v}
-datadir = mysql_params['--datadir']
-config_file = mysql_params['--defaults-file']
-binlog_dir = File.dirname(%x{cat #{config_file} |egrep '^log-bin'|grep -v '.index'|awk -F= '{print $2}'}.gsub(/\s+/,'').chomp)
-
-
-# If master-bin.000001 exists then only purge logs if disk space is constrained
-binary_log_name = %x{cat #{config_file}|grep '^log-bin'|grep -v 'index' |awk -F= '{print $2}'}.gsub(/\s/,'')
-if binary_log_name == ''
-	log_info "log-bin not set in config file, host does not have master role, unable to proceed"
-	exit(0) 
-end
-if File.exists?(binary_log_name + '.000001')
-  purge_threshold = 90 
-  if %x{df -h |egrep "/db" |awk '{print $5}'}.to_i < purge_threshold
-    log_info "The first binary log exists and the purge threshold has not been reached; skipping purge action"
-    exit(0)
-  end
-end
-
-# Check master for all connected replication slaves
-result = run_query('localhost', dbuser, dbpassword, 'show processlist')
-slave_hosts = []
-min_log = 0 
-result.each do |line|
-  if line.include? 'Binlog Dump'
-    slave = Hash.new
-    slave["hostname"] = extract_host(line)
-    slave["Relay_Master_Log_File"] = slave_log_file(slave["hostname"], dbuser, dbpassword)
-    slave["Relay_Master_Log_File"] =~ /\w+.(\d{6})/ and min_log = $1.to_i if $1.to_i < min_log || min_log == 0
-    log_info "Slave Hostname: #{slave["hostname"]}, Relay_Master_log: #{slave["Relay_Master_Log_File"]}"
-    slave_hosts << slave
-  end
-end
-
-# stop log purge #{keep_logs} logs before the current read position
-stop_log = min_log - keep_logs
-
-# if standalone master and no replicas are found we stop purge #{keep_logs} logs before master's current position
-if min_log == 0 and File.exists?(chef_file)
-  chef_config = JSON.parse(File.read(chef_file))
-  if chef_config['db_slaves'].empty? or chef_config['db_slaves'].nil?
-    current_master = %x{cd #{binlog_dir} && ls -tr master-bin.[0-9]* | tail  -n 1}
-    current_master =~ /\w+.(\d{6})/ and stop_log = $1.to_i + 1 - keep_logs
-  elsif min_log == 0
-    log_error "Slave is on record as '#{chef_config['db_slaves']}' but replication is not running."
-    exit 1
-  end
-end
-
-
-# Purge logs based on minimum position of all servers
-min_master_log = %x{cd #{binlog_dir} && ls -tr master-bin.[0-9]* | head -n 1}
-min_master_log =~ /\w+.(\d{6})/ and min_master_num = $1.to_i + 1
-
-min_master_num.upto(min_master_num + 10) do |i|
-# purge up to 10 files as long as the top file is less than the minimum replica log
-  if stop_log < 0
-    log_error "Could not verify replication status, confirm that replication is running.  Exiting!"
-    break
-  elsif i >= stop_log + 1
-    log_info "File number of #{i} exceeds minimum purge file of #{stop_log + 1} based on keeping #{keep_logs} files.  Exiting!"
-    break
-  end
-  file = "master-bin.%06d" % i
-  log_info "Purging binary logs to #{file}"
-  run_query('localhost', dbuser, dbpassword, "purge master logs to '#{file}'")
-  sleep 120
-end
-
-log_info Time.now