snapshot_reload 1.0.0

data/lib/snapshot_reload/reload.rb

@@ -0,0 +1,69 @@
+=begin
+
+   Issue the actual commands to drop, create, and load the database
+
+   +Copyright:+ (c) 2012, Novu, LLC
+   +License:+ All rights reserved. For internal and private use only.
+   +Author:+ Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+require 'methadone'
+
+module SnapshotReload
+  class SnapshotReload
+    include Methadone::CLILogging
+    # include Methadone::CLILogger
+    include Methadone::SH
+
+    def reload
+      drop_db
+      create_db
+      load_db
+    end
+
+    def drop_db
+      mysqladmin_go("drop")
+    end # method drop
+
+    def create_db
+      mysqladmin_go("create")
+    end
+
+    def mysqladmin_go(command)
+      return false unless "drop create".include?(command)
+      passopt = (@password.nil? or @password.empty?) ? '' : "--password=#{@password}"
+      cmd = "mysqladmin --host=#{@host} --user=#{@username} #{passopt} --force #{command} #{@database}"
+      run_it(cmd)
+    end # mysqladmin_go method
+
+    def load_db
+      passopt = (@password.nil? or @password.empty?) ? '' : "--password=#{@password}"
+      fetch_snapshot
+      cmd = "gunzip < #{self.sql_file} | mysql --host=#{@host} --user=#{@username} #{passopt} #{@database}"
+      run_it(cmd)
+    end
+
+    def run_it(cmd)
+      info("Dry-run, no commands will be executed.") if @dry_run and @verbose
+      info("Command issued: #{cmd}") if @verbose
+      unless @dry_run
+        sh cmd do |stdout, stderr, retval|
+          if retval != 0
+            unless cmd.match("mysqladmin.*drop") and stderr.match("database doesn't exist")
+              fatal("mysqladmin drop command failed:")
+              fatal(stdout)
+              fatal(stderr)
+              exit(EDROPFAILED)
+            else
+              
+            end
+          end
+        end
+      else
+        true
+      end
+    end
+
+  end
+end

data/lib/snapshot_reload/validate.rb

@@ -0,0 +1,111 @@
+=begin
+
+Methods to validate input
+
++Copyright:+ (c) 2012, Novu, LLC
++License:+ All rights reserved. For internal and private use only.
++Author:+ Tamara Temple <tamara.temple@novu.com>
+
+=end  
+require 'methadone'
+
+module SnapshotReload
+
+  class SnapshotReload
+    include Methadone::CLILogging
+
+
+    def validate_configuration(config_file)
+
+      unless File.exists?(config_file)
+        fatal("#{config_file} file does not exist") 
+        exit(ENOCONFIG)
+      end
+
+      config = YAML.load(File.read(config_file))
+      unless config.class == Hash
+        fatal("#{config_file} is not YAML")
+        exit(ECONFIGNOTYAML)      
+      end
+
+      debug("Config: #{config.to_s}")
+
+      config
+    end
+
+    def validate_environment(env,config)
+      env ||= ENV['RAILS_ENV'] ||= DEFAULT_ENVIRONMENT
+      unless config.has_key?(env)
+        fatal("#{env} not found in configuration")
+        exit(ENOENVINCONFIG)
+      end
+
+      debug("Env: #{env}")
+      env
+    end
+
+    def validate_source(source)
+      source ||= DEFAULT_S3_SOURCE
+    end
+
+    def validate_aws(aws_conf, aws_key = nil, aws_secret = nil)
+
+      # DONE: fix scenarios so they run this code (pass in an S3 source!!)
+
+      # aws_key and/or aws_secret were not nil -- therefore specified on the command line
+
+      # **Both** aws_key and aws_secret must be given.
+      # if only one but not the other is given, it's an error
+
+      debug("Is aws_key present? #{aws_key.present? ? "yes" : "no"} key: #{aws_key}")
+      debug("Is aws_secret present? #{aws_secret.present? ? "yes" : "no"} secret: #{aws_secret}")
+
+      if (aws_key.present? ^ aws_secret.present?) # exclusive or, one must be true, but not the other
+        fatal("Must provide *both* aws-key and aws-secret")
+        exit(EMISSINGKEYORSECRET)  
+      end
+
+      if aws_key.nil? and aws_secret.nil?
+
+        aws_conf ||= DEFAULT_AWS_CREDS
+
+        if File.exists?(aws_conf)
+          File.open(aws_conf, 'r') do |awsfile|
+            awsfile.each_line do |line|
+              m = line.match('^access_key\s*=\s*(.*)$')
+              aws_key = m[1] if m
+
+              m = line.match('^secret_key\s*=\s*(.*)$')
+              debug("secret_key match -> #{m[0]} -> #{m[1]}") if m
+              aws_secret = m[1] if m
+            end
+          end
+        else
+          fatal("#{aws_conf} does not exist!")
+          exit(ENOCRED)
+        end
+
+      end
+
+      debug("key: #{aws_key}\nsecret: #{aws_secret}")
+      [ aws_key, aws_secret ]
+
+    end
+
+    def check_field(conf,env,field,required=true)
+      if conf[env].has_key?(field)
+        conf[env][field]
+      else
+        if required
+          fatal("Must provide #{field} in configuration")
+          exit(ENOFIELD)
+        else
+          nil
+        end
+      end
+    end
+
+  end # class
+
+end # module
+

data/lib/snapshot_reload/version.rb

@@ -0,0 +1,18 @@
+=begin
+
+  Set Application version info
+
+   +Copyright:+ (c) 2012 Novu, LLC
+   +License:+ All rights reserved. For internal and private use only.
+   +Author:+ Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+
+
+module SnapshotReload
+  VERSION = "1.0.0"
+  SUMMARY = %q{Redeploy the obfuscated data from production to other servers}
+  DESCRIPTION =  %q{Take the obfuscated production data from it's nightly backup and place it into a working database. This script is run nightly to put obfuscated data into the Q* test server environments.}
+
+end

data/snapshot_reload.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'snapshot_reload/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "snapshot_reload"
+  gem.version       = SnapshotReload::VERSION
+  gem.authors       = ["Tamara Temple"]
+  gem.email         = ["tamara.temple@novu.com"]
+  gem.description   = SnapshotReload::DESCRIPTION
+  gem.summary       = SnapshotReload::SUMMARY
+  gem.homepage      = "https://sites.google.com/a/novu.com/hq/home/technology-operations/systems-engineering/database-refresh-script"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency('rdoc')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('rake', '~> 0.9.2')
+  gem.add_development_dependency('rspec')
+  gem.add_dependency('methadone', '~> 1.2.3')
+  gem.add_dependency('fog') # for AWS S3 access
+end

data/spec/snapshot_reload_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+
+module SnapshotReload
+
+  describe "SnapshotReload" do
+    before(:each) do
+      @x_yml = "/tmp/x.yml"
+      x_yml_file = File.open(@x_yml,'w')
+      x_yml_file.puts <<-END
+qa:
+  adapter: mysql2
+  host: localhost
+  database: novuqa
+  username: novuadmin
+  password:
+other:
+  adapter: mysql2
+  host: localhost
+  database: novuqa
+  username: novuadmin
+  password:
+      END
+      x_yml_file.close
+
+      @aws_conf = "/tmp/aws_conf"
+      aws_conf_file = File.open(@aws_conf, 'w')
+      aws_conf_file.puts <<-END
+access_key = 12345
+secret_key = 67890
+      END
+      aws_conf_file.close
+
+      @options = Hash.new
+      @options[:e] = @options['e'] = @options[:env] = @options['env'] = DEFAULT_ENVIRONMENT
+      @options[:s] = @options['s'] = @options[:source] = @options['source'] = 'testdata/novu_2012-12-11.sql.gz'
+      # @options[:s] = @options['s'] = @options[:source] = @options['source'] = 'xxx'
+      @options[:c] = @options['c'] = @options[:"aws-conf"] = @options['aws-conf'] = @aws_conf
+      @options[:n] = @options['n'] = @options[:"dry-run"] = @options['dry-run'] = true
+      @options[:v] = @options['v'] = @options[:verbose] = @options['verbose'] = false
+
+    end
+
+    it "can be instantiated" do
+      SnapshotReload.new(@x_yml,@options).should be_an_instance_of(SnapshotReload)
+    end
+
+    it "returns a hash for #config" do
+      SnapshotReload.new(@x_yml,@options).config.should be_an_instance_of(Hash)
+    end
+
+    it "environment is qa" do
+      SnapshotReload.new(@x_yml,@options).env.should eq('qa')
+    end
+
+    it "responds to drop_db" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('drop_db').should be_true
+    end
+
+    it "responds to create_db" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('create_db').should be_true
+    end
+
+    it "responds to mysqladmin_go" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('mysqladmin_go').should be_true
+    end
+
+    it "responds to load_db" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('load_db').should be_true
+    end
+
+
+    it "responds to fetch_snapshot" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('fetch_snapshot').should be_true
+    end
+
+    it "sets @sql_file" do
+      @sn = SnapshotReload.new(@x_yml,@options)
+      @sn.fetch_snapshot
+      @sn.sql_file.should eq("testdata/novu_2012-12-11.sql.gz")
+    end
+
+    it "responds to run_it" do
+      SnapshotReload.new(@x_yml,@options).respond_to?('run_it').should be_true
+    end
+
+    it "emulates drop command" do
+      SnapshotReload.new(@x_yml,@options).drop_db.should be_true
+    end
+
+    it "emulates create command" do
+      SnapshotReload.new(@x_yml,@options).create_db.should be_true
+    end
+
+    it "emulates load command" do
+      @sn = SnapshotReload.new(@x_yml,@options)
+      @sn.load_db.should be_true
+    end
+
+
+
+    after(:each) do
+      File.unlink(@x_yml)
+      File.unlink(@aws_conf)
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'snapshot_reload'

metadata

@@ -0,0 +1,178 @@
+--- !ruby/object:Gem::Specification
+name: snapshot_reload
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Tamara Temple
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: methadone
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Take the obfuscated production data from it's nightly backup and place
+  it into a working database. This script is run nightly to put obfuscated data into
+  the Q* test server environments.
+email:
+- tamara.temple@novu.com
+executables:
+- snapshot_reload
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- README.rdoc
+- Rakefile
+- bin/snapshot_reload
+- features/snapshot_live.feature
+- features/snapshot_reload.feature
+- features/step_definitions/snapshot_live_steps.rb
+- features/step_definitions/snapshot_reload_steps.rb
+- features/support/env.rb
+- lib/snapshot_reload.rb
+- lib/snapshot_reload/String.rb
+- lib/snapshot_reload/defaults.rb
+- lib/snapshot_reload/errors.rb
+- lib/snapshot_reload/fetch.rb
+- lib/snapshot_reload/reload.rb
+- lib/snapshot_reload/validate.rb
+- lib/snapshot_reload/version.rb
+- snapshot_reload.gemspec
+- spec/snapshot_reload_spec.rb
+- spec/spec_helper.rb
+homepage: https://sites.google.com/a/novu.com/hq/home/technology-operations/systems-engineering/database-refresh-script
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Redeploy the obfuscated data from production to other servers
+test_files:
+- features/snapshot_live.feature
+- features/snapshot_reload.feature
+- features/step_definitions/snapshot_live_steps.rb
+- features/step_definitions/snapshot_reload_steps.rb
+- features/support/env.rb
+- spec/snapshot_reload_spec.rb
+- spec/spec_helper.rb
+has_rdoc: