ey_cloud_server 1.4.28 → 1.4.29

data/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+pkg
+vendor/gems/ruby/1.8/*
+vendor/gems/environment.rb
+!vendor/gems/ruby/1.8/cache
+gbin
+coverage
+spec/config.yml
+tmp
+.bundle

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.8.7

data/Gemfile

@@ -0,0 +1,22 @@
+source 'http://rubygems.org/'
+source 'http://geminst:hoa3laml@rubygems.engineyard.com/'
+
+gemspec
+
+## Group Dependencies
+
+group :test do
+  gem 'rspec'
+  gem 'randexp'
+  gem 'cucumber'
+  gem 'rcov'
+  gem 'fakeweb', '1.2.6.ninjas'
+  gem 'fakefs', '=0.0.1.ninjas'
+  gem 'dbi'
+  gem 'dbd-mysql'
+  gem 'ruby-debug', :platforms => :mri
+end
+
+group :development do
+  gem 'rake'
+end

data/Gemfile.lock

@@ -0,0 +1,124 @@
+PATH
+  remote: .
+  specs:
+    ey_cloud_server (1.4.29.pre)
+      ey_enzyme
+      ey_instance_api_client (~> 0.1.5)
+      fog (~> 1.1.2)
+      json (~> 1.6.4)
+      open4 (~> 1.3.0)
+
+GEM
+  remote: http://rubygems.org/
+  remote: http://geminst:hoa3laml@rubygems.engineyard.com/
+  specs:
+    builder (3.0.0)
+    chef (0.6.0.2)
+      erubis
+      extlib
+      json
+      ohai
+      ruby-openid
+      stomp
+    chef-deploy (0.2.5)
+    columnize (0.3.6)
+    cucumber (1.1.4)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.2)
+      gherkin (~> 2.7.1)
+      json (>= 1.4.6)
+      term-ansicolor (>= 1.0.6)
+    dbd-mysql (0.4.4)
+      dbi (>= 0.4.0)
+      mysql
+    dbi (0.4.5)
+      deprecated (= 2.0.1)
+    deprecated (2.0.1)
+    diff-lcs (1.1.3)
+    erubis (2.7.0)
+    excon (0.9.4)
+    extlib (0.9.15)
+    ey_enzyme (0.9.44)
+      chef (= 0.6.0.2)
+      chef-deploy (= 0.2.5)
+      json
+      rest-client (~> 1.6.1)
+    ey_instance_api_client (0.1.5)
+      json
+      rest-client
+    fakefs (0.0.1.ninjas)
+    fakeweb (1.2.6.ninjas)
+    fog (1.1.2)
+      builder
+      excon (~> 0.9.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0.3)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.1)
+    gherkin (2.7.1)
+      json (>= 1.4.6)
+    json (1.6.4)
+    linecache (0.46)
+      rbx-require-relative (> 0.0.4)
+    mime-types (1.17.2)
+    mixlib-cli (1.2.2)
+    mixlib-config (1.1.2)
+    mixlib-log (1.3.0)
+    multi_json (1.0.4)
+    mysql (2.8.1)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.2.1)
+    nokogiri (1.5.0)
+    ohai (0.6.10)
+      mixlib-cli
+      mixlib-config
+      mixlib-log
+      systemu (~> 2.2.0)
+      yajl-ruby
+    open4 (1.3.0)
+    rake (0.9.2.2)
+    randexp (0.1.6)
+    rbx-require-relative (0.0.5)
+    rcov (0.9.11)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    ruby-hmac (0.4.0)
+    ruby-openid (2.1.8)
+    stomp (1.2.0)
+    systemu (2.2.0)
+    term-ansicolor (1.0.7)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber
+  dbd-mysql
+  dbi
+  ey_cloud_server!
+  fakefs (= 0.0.1.ninjas)
+  fakeweb (= 1.2.6.ninjas)
+  rake
+  randexp
+  rcov
+  rspec
+  ruby-debug

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Engine Yard Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,46 @@
+== ey
+
+AKA Ey Instance Tools
+
+= Development
+
+Requirements:
+
+    bundler
+    gnupg
+    postgres and mysql servers
+
+Then bundle everything up:
+
+    bundle
+
+Configure your system:
+
+    cp spec/config-example.yml spec/config.yml
+
+    edit the database users
+    add amazon credentials to use for uploading test backups to S3
+
+== Testing
+
+To run the specs:
+
+    bundle exec rake
+
+To run a specifc spec:
+
+    bundle exec spec -f spec/ey_backup/cli_spec.rb
+
+Running the bin files are easy too: 
+
+    bin/ey-recipes
+
+== Releasing
+
+    ey-gem release
+
+Bump the cloud_cookbooks version in dracul/resin:
+
+    cd ../dracul
+    edit lib/dracul.rb and bump the versions in the ey resin config
+    follow the instructions in the dracul readme from here to release a new dracul

data/Rakefile

@@ -0,0 +1,19 @@
+Bundler.setup(:runtime, :test)
+
+$:.unshift File.dirname(__FILE__) + '/lib'
+require File.dirname(__FILE__) + '/lib/ey-flex'
+
+require 'rspec/core/rake_task'
+desc "Run unit specifications"
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+  t.rspec_opts = %w(-fs --color --backtrace)
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new do |t|
+  t.cucumber_opts = %w{--format pretty}
+end
+
+desc "Run unit specifications"
+task :default => [:spec, :cucumber]

data/TODO

@@ -0,0 +1,4 @@
+TODO:
+Fix LICENSE with your name
+Fix Rakefile with your name and contact info
+Add your code to lib/<%= name %>.rb

data/ey_cloud_server.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/ey_cloud_server/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["EngineYard"]
+  gem.email         = ["engineering@engineyard.com"]
+  gem.description   = %q{}
+  gem.summary       = %q{Server side components for Engine Yard's cloud}
+  gem.homepage      = ""
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "ey_cloud_server"
+  gem.require_paths = ["lib"]
+  gem.version       = EY::CloudServer::VERSION
+
+  gem.add_runtime_dependency 'json', '~> 1.6.4'
+  gem.add_runtime_dependency 'open4', "~> 1.3.0"
+
+  gem.add_runtime_dependency 'fog', '~> 1.1.2'
+  gem.add_runtime_dependency "ey_enzyme"
+
+  gem.add_runtime_dependency "ey_instance_api_client", "~> 0.1.5"
+end

data/features/downloading_a_mysql_backup.feature

@@ -0,0 +1,12 @@
+Feature:
+  Scenario: a single database
+    Given there is a mysql database (first)
+    And backups for mysql are run
+    When backup "0" for the "first" database is downloaded
+    Then there is a backup file for the "first" database
+
+  Scenario: no database backup
+    Given there is a mysql database (first)
+    When backup "0" for the "first" database is downloaded
+    Then there is no backup file for the "first" database
+    And stderr contains "No backup found for database"

data/features/making_a_postgresql_backup.feature

@@ -0,0 +1,12 @@
+Feature:
+  Scenario: no backups run
+    Given there is a postgresql database (first)
+    When listing the backups for the "first" postgresql database
+    Then the output contains "0 backup(s) found"
+
+  Scenario: a single backup run
+    Given there is a postgresql database (first)
+    And backups for postgresql are run
+    When listing the backups for the "first" postgresql database
+    Then the output contains "1 backup(s) found"
+    And the output contains "0:%s" substituting the "first" postgresql database

data/features/restoring_postgres_backup.feature

@@ -0,0 +1,12 @@
+Feature:
+  Scenario: no backup
+    Given there is a postgresql database (first)
+    When backup "0" for the "first" database is restored
+    Then stderr contains "No backup found for database"
+
+  Scenario: with a backup
+    Given there is a postgresql database (first)
+    And backups for postgresql are run
+    And the "first" database is dropped
+    When backup "0" for the "first" database is restored
+    Then the "first" postgresql database exists

data/features/step_definitions/backups.rb

@@ -0,0 +1,61 @@
+Given \
+    /^backups for mysql are run$/ do
+  capturing_stdio do
+    EY::Backup.logger = EY::Backup::Logger.new
+    EY::Backup.run([ "-c", backup_config_file ])
+  end
+end
+
+Given \
+    /^backups for postgresql are run$/ do
+  capturing_stdio do
+    EY::Backup.logger = EY::Backup::Logger.new
+    EY::Backup.run(["-e", "postgresql", "-c", backup_config_file ])
+  end
+end
+
+When \
+    /^backup "([^\"]*)" for the "([^\"]*)" database is downloaded$/ do |index,db_key|
+  db_name = created_mysql_dbs[db_key]
+  capturing_stdio do
+    EY::Backup.logger = EY::Backup::Logger.new
+    EY::Backup.run(["-d", "#{index}:#{db_name}", "-c", backup_config_file ])
+  end
+end
+
+When \
+    /^backup "([^\"]*)" for the "([^\"]*)" database is restored$/ do |index,db_key|
+  db_name = created_postgresql_dbs[db_key]
+  capturing_stdio do
+    EY::Backup.logger = EY::Backup::Logger.new
+    EY::Backup.run(["-e", "postgresql", "-r", "#{index}:#{db_name}", "-c", backup_config_file ])
+  end
+end
+
+When \
+    /^listing the backups for the "([^\"]*)" postgresql database$/ do |db_key|
+  db_name = created_postgresql_dbs[db_key]
+  capturing_stdio do
+    EY::Backup.logger = EY::Backup::Logger.new
+    EY::Backup.run(["-e", "postgresql", "-l", db_name, "-c", backup_config_file ])
+  end
+end
+
+Then \
+    /^the output contains "([^\"]*)" substituting the "([^\"]*)" postgresql database$/ do |text,db_key|
+  db_name = created_postgresql_dbs[db_key]
+  last_stdout.should include(text % db_name)
+end
+
+Then \
+    /^there is a backup file for the "([^\"]*)" database$/ do |db_key|
+  db_name = created_mysql_dbs[db_key]
+  Dir.glob("#{EY::Backup.tmp_dir}/#{db_name}.*").should have(1).entries
+  Dir.glob("#{EY::Backup.tmp_dir}/#{db_name}.*").each{ |f| FileUtils.rm(f) }
+end
+
+Then \
+    /^there is no backup file for the "([^\"]*)" database$/ do |db_key|
+  db_name = created_mysql_dbs[db_key]
+  Dir.glob("#{db_name}.*").should have(0).entries
+end

data/features/step_definitions/commands.rb

@@ -0,0 +1,9 @@
+Then \
+    /^the output contains "([^\"]*)"$/ do |text|
+  last_stdout.should include(text)
+end
+
+Then \
+    /^stderr contains "([^\"]*)"$/ do |text|
+  last_stderr.should include(text)
+end

data/features/step_definitions/mysql.rb

@@ -0,0 +1,4 @@
+Given \
+    /^there is a mysql database \(([^\)]*)\)$/ do |db_key|
+  create_mysql_database(db_key)
+end

data/features/step_definitions/postgresql.rb

@@ -0,0 +1,15 @@
+Given \
+    /^there is a postgresql database \(([^\)]*)\)$/ do |db_key|
+  create_postgresql_database(db_key)
+end
+
+Then \
+    /^the "([^\"]*)" postgresql database exists$/ do |db_key|
+  db_name = created_postgresql_dbs[db_key]
+  check_postgresql_database(db_name)
+end
+
+Given \
+    /^the "([^\"]*)" database is dropped$/ do |db_key|
+  drop_postgresql_database(db_key)
+end

data/features/support/env.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + "/../../lib/ey-flex"
+require File.dirname(__FILE__) + "/../../lib/ey_backup"
+require 'fakeweb'
+require 'fakefs/safe'
+require 'randexp'
+
+require File.dirname(__FILE__) + '/../../spec/fakefs_hax'
+require File.dirname(__FILE__) + '/../../spec/helpers'
+
+FakeWeb.allow_net_connect = false # if it's not here, it doesn't exist
+Randexp::Dictionary.words
+Fog.mock!
+
+Fog.mock!
+
+World(Helpers)
+
+Before do
+  run_before
+end
+
+After do
+  run_after
+end

data/lib/ey-flex/big-brother.rb

@@ -55,14 +55,14 @@ module EY
       end
       @result.to_json
     end
-    
+
     def check_mysql
       check_process('mysqld')
       DBI.connect("DBI:Mysql:mysql:#{@dna['db_host']}", 'root', @dna['users'].first['password'])
     rescue DBI::DatabaseError => e
       @result['mysqld'] = 'down'
     end
-    
+
     def check_process(name)
       return if name == ''
       pids = `pgrep #{name}`.split("\n")

data/lib/ey-flex.rb

@@ -1,11 +1,9 @@
 require 'json'
-require 'aws/s3'
 require 'date'
 require 'digest'
 require 'net/http'
 require 'fileutils'
 require 'json/ext'
-require 'right_aws'
 require 'open-uri'
 require 'rest_client'
 require 'dbi'
@@ -49,5 +47,4 @@ require lib_dir + '/big-brother'
 require lib_dir + '/bucket_minder'
 require lib_dir + '/ey-api'
 require lib_dir + '/snapshot_minder'
-require lib_dir + '/ec2'
 require lib_dir + '/../ey_cloud_server/version'

data/lib/ey_backup/cli.rb

@@ -61,7 +61,7 @@ module EY
             options[:db] = db
           end
 
-          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres, mongodb.") do |engine|
+          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
             options[:engine] = engine
           end
 

data/lib/ey_backup/engines/postgresql_engine.rb

@@ -6,7 +6,7 @@ module EY
       def dump(database_name, basename)
         file = basename + '.dump'
 
-        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --format=c --no-owner --no-privileges -Upostgres #{database_name}"
+        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --format=c --no-owner --no-privileges -U#{username} #{database_name}"
 
         if gpg?
           command << " | " << GPGEncryptor.command_for(key_id)
@@ -33,7 +33,7 @@ module EY
           raise "Cannot load a GPG backup"
         end
 
-        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c --ignore-version -Upostgres -d #{database_name}"
+        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c --ignore-version -U#{username} -d #{database_name}"
 
         run(command)
       end

data/lib/ey_backup.rb

@@ -126,7 +126,6 @@ require lib_dir + '/backend'
 
 require lib_dir + '/engines/mysql_engine'
 require lib_dir + '/engines/postgresql_engine'
-require lib_dir + '/engines/mongodb_engine'
 
 require lib_dir + '/processors/gzipper'
 require lib_dir + '/processors/splitter'

data/lib/ey_cloud_server/version.rb

@@ -1,3 +1,5 @@
-module EY::CloudServer
-  VERSION = '1.4.28'
+module EY
+  module CloudServer
+    VERSION = '1.4.29'
+  end
 end

data/script/ci

@@ -0,0 +1,31 @@
+#! /bin/sh
+
+set -x
+set -e # exits on any non 0 exit code
+
+# TODO: WHY?
+
+sudo cp -pr /root/.gnupg $HOME/
+sudo chown -R deploy:deploy $HOME/.gnupg/
+
+cp spec/config-example.yml spec/config.yml
+
+sed -i "s/^mysql_host: *.*$/mysql_host: $DB_HOST/g" spec/config.yml
+sed -i "s/^mysql_user: *.*$/mysql_user: root/g" spec/config.yml
+sed -i "s/^mysql_password: *.*$/mysql_password: $DB_PASS/g" spec/config.yml
+
+sed -i "s/^postgresql_host: *.*$/postgresql_host: $DB_HOST/g" spec/config.yml
+sed -i "s/^postgresql_user: *.*$/postgresql_user: $USER/g" spec/config.yml
+sed -i "s/^postgresql_password: *.*$/postgresql_password: $DB_PASS/g" spec/config.yml
+
+# Get the real aws keys; first remove what we have
+grep -v aws_ spec/config.yml >spec/config.yml.tmp
+mv spec/config.yml.tmp spec/config.yml
+sudo grep -i aws_secret_key /etc/chef/dna.json | tail -n 1 | sed -e 's/,* *$//' -e 's/^ *//' >>spec/config.yml
+sudo grep -i aws_secret_id /etc/chef/dna.json | tail -n 1 | sed -e 's/,* *$//' -e 's/^ *//' >>spec/config.yml
+
+echo "CONFIG FILE: "
+cat spec/config.yml
+
+bundle install --deployment &> bundle.log
+bundle exec rake --trace

data/spec/config-example.yml

@@ -7,8 +7,5 @@ mysql_host: localhost
 postgresql_user: root
 postgresql_password:
 postgresql_host: localhost
-mongodb_user: root
-mongodb_passwd: 
-mongodb_host: localhost
 aws_secret_id: 'abcd123.id'
 aws_secret_key: 'abcd123.key'

data/spec/ey_backup/postgres_backups_spec.rb

@@ -12,7 +12,7 @@ describe "Postgres Backups" do
 
     EY::Backup.run(["-c", backup_config_file, "-l", @db_name, "-e", "postgresql"])
 
-    stdout.should match(/0:#{@db_name}.*\.dump.gz$/)
+    stdout.should match(/0:#{@db_name}.*\.dump$/)
   end
 
   it "makes a split backup" do