snapshot_reload 1.0.0

data/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+results.html
+pkg
+html
+*.sublime-*
+/testdata/
+/tmp/
+*.gz
+README.html

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p327@snapshot_reload"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.16.20 ()" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in snapshot_reload.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    snapshot_reload (0.0.1)
+      fog
+      methadone (~> 1.2.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aruba (0.5.1)
+      childprocess (~> 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    builder (3.1.4)
+    childprocess (0.3.6)
+      ffi (~> 1.0, >= 1.0.6)
+    cucumber (1.2.1)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.11.0)
+      json (>= 1.4.6)
+    diff-lcs (1.1.3)
+    excon (0.13.4)
+    ffi (1.2.0)
+    fog (1.3.1)
+      builder
+      excon (~> 0.13.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.0.4)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.1)
+    gherkin (2.11.5)
+      json (>= 1.4.6)
+    json (1.7.5)
+    methadone (1.2.3)
+      bundler
+    mime-types (1.19)
+    multi_json (1.3.7)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.5.2)
+    nokogiri (1.5.5)
+    rake (0.9.5)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.1)
+    rspec-expectations (2.12.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.0)
+    ruby-hmac (0.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+  rake (~> 0.9.2)
+  rdoc
+  rspec
+  snapshot_reload!

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Tamara Temple
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,114 @@
+# SnapshotReload
+
+SnapshotReload is a ruby gem that pulls down a gzipped database snapshot, and
+reloads it into a database.
+
+This gem is part of the suite of tools needed to bring up the Q* server environments
+for nightly testing of the novu web site code using obfuscated production data
+(All participants' personal data has been scrubbed.)
+
+The gem will work during the nightly chef deploy to the Q* servers.
+
+## Installation
+
+Clone the repo from github:
+
+        $ git clone git@github.com:novu/snapshot_reload.git
+
+Then, install the gem:
+
+        $ gem install snapshot_reload
+
+## Usage
+
+The gem has a complete usage output, run snapshot_reload -h for the current listing.
+
+This is what it looks like at version 1.0.0:
+
+    Usage: snapshot_reload [options] config
+
+    Redeploy the obfuscated data from production to other servers
+
+    v1.0.0
+
+    Options:
+        -h, --help                       Show command line help
+        -e, --env ENVIRONMENT            Environment to use in configuration file
+                                         Will use ENV['RAILS_ENV'] if not specified and the envar exists
+                                         Otherwise will use qa
+        -s, --source S3SOURCE            Provide the AWS S3 source of the obfuscated data
+                                         Optional flag. Default value is s3://novu_backups/db_backups/clean-mysqldump.sql.gz
+        -c, --aws-conf AWS_CONFIG        Provide an alternative AWS configuration file containing log in credentials
+                                         Optional flag. Default value is /opt/novu/.s3cfg
+        -k, --aws-key AWSKEY             Provide the AWS key for logging in. Overrides value in AWS_CONFIG and default if given
+                                         Required if --aws-secret is specified
+        -y, --aws-secret AWSSECRET       Provide the AWS secret key for logging in. Overrides value in AWS_CONFIG and default if given.
+                                         Required if --aws-key is specified.
+        -n, --dry-run                    Don't run anything, just show what would be run
+        -v, --verbose                    Provide lots of information about what is happening
+        -q, --quiet                      Be completely silent. Overrides --verbose switch
+            --version                    Show help/version info
+            --log-level LEVEL            Set the logging level
+                                         (debug|info|warn|error|fatal)
+                                         (Default: info)
+
+    Arguments:
+
+        config
+            Configuration file in same format as Rails config/database.yml file
+
+## Discussion
+
+This gem implements the steps necessary to stand up an obfuscated copy of the production database for http://www.novu.com in the Q* server
+environments (Qdev, Qrel, and Qnext). It is meant to be installed with the chef deployments for these servers. It can also be used to get any 
+gzipped snapshot of the application database (does not have to be obfuscated production data) and stand it up to any database environment.
+This makes it useful for individual developers and testers, as it can stand up a database on a local machine, or another server environment,
+independently of chef. All you need is the gzipped file as `source`, and the database credentials in `config`.
+
+## Examples
+
+### The default, standing up the Qdev server via chef recipe:
+    
+        $ snapshot_reload /opt/novu/current/config/database.yml
+
+The standard location of the application's database configuration file is given, the RAILS_ENV is set to 'qa' and the default S3
+credentials are stored in /opt/novu/.s3cfg 
+
+### Install a snapshot of nightly cleaned production data into your local database
+
+You will need to obtain S3 credentials that give you access to the (non-production) buckets, see nvse@novu.com to obtain if you don't
+have any. Put them into a file that looks like a standard .ini key = value file:
+
+        access_key = YOURAWSACCESSKEY
+        secret_key = YOURAWSSECRETACCESSKEY
+
+No quotes, etc, just the values. The keys must be given as shown, at the start of the line. White space around the '=' is optional.
+
+Then issue:
+
+        $ snapshot_reload -c path/to/aws/credentials -e development path/to/database.yml
+
+and the data will be loaded from the S3 bucket into the database specified by the development environment in the configuration file.
+
+### If you have a local gzipped copy of data to load, you can specify that as well:
+
+        $ snapshot_reload --source path/to/sql.gz -e development path/to/database.yml
+
+This will load the data in the sql.gz file into the database specified by the 'development' environment in the configuration file.
+
+
+## Author
+
+Tamara Temple <tamara.temple@novu.com>
+
+## Copyright, Licensing
+
+Copyright &copy; Novu, LLC. All rights reserverd. For internal and private use only.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,19 @@
+= snapshot_reload - DESCRIBE YOUR GEM
+
+Author::  YOUR NAME (YOUR EMAIL)
+Copyright:: Copyright (c) 2012 YOUR NAME
+
+
+DESCRIBE YOUR GEM HERE
+
+== Links
+
+* {Source on Github}[LINK TO GITHUB]
+* RDoc[LINK TO RDOC.INFO]
+
+== Install
+
+== Examples
+
+== Contributing
+

data/Rakefile

@@ -0,0 +1,66 @@
+def dump_load_path
+  puts $LOAD_PATH.join("\n")
+  found = nil
+  $LOAD_PATH.each do |path|
+    if File.exists?(File.join(path,"rspec"))
+      puts "Found rspec in #{path}"
+      if File.exists?(File.join(path,"rspec","core"))
+        puts "Found core"
+        if File.exists?(File.join(path,"rspec","core","rake_task"))
+          puts "Found rake_task"
+          found = path
+        else
+          puts "!! no rake_task"
+        end
+      else
+        puts "!!! no core"
+      end
+    end
+  end
+  if found.nil?
+    puts "Didn't find rspec/core/rake_task anywhere"
+  else
+    puts "Found in #{path}"
+  end
+end
+require 'bundler'
+require 'rake/clean'
+
+begin
+require 'rspec/core/rake_task'
+rescue LoadError
+dump_load_path
+raise
+end
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+RSpec::Core::RakeTask.new do |t|
+  # Put spec opts in a file named .rspec in root
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+  
+  rd.main = "README.rdoc"
+  
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:spec,:features]
+

data/bin/snapshot_reload

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+=begin
+
+   Main script for snapshot_reload
+
+   +Copyright:+ (c) 2012, Novu, LLC
+   +License:+ All rights reserved. For internal and private use only.
+   +Author:+ Tamara Temple <tamara.temple@novu.com>
+
+  Makes use of methadone gem
+  
+=end
+
+# TODO: expand help text
+
+require 'optparse'
+require 'methadone'
+require 'snapshot_reload'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |config|
+    debug("config: #{config.to_s}")
+    debug("options: #{options.to_s}")
+    snapshot_reload = SnapshotReload::SnapshotReload.new(config, options)
+  end
+
+  description SnapshotReload::SUMMARY
+
+  on("-e", "--env ENVIRONMENT", "Environment to use in configuration file",
+    "Will use ENV['RAILS_ENV'] if not specified and the envar exists",
+    "Otherwise will use #{SnapshotReload::DEFAULT_ENVIRONMENT}")
+
+  on("-s", "--source S3SOURCE", "Provide the AWS S3 source of the obfuscated data",
+    "Optional flag. Default value is #{SnapshotReload::DEFAULT_S3_SOURCE}")
+
+  on("-c", "--aws-conf AWS_CONFIG",
+    "Provide an alternative AWS configuration file containing log in credentials",
+    "Optional flag. Default value is #{SnapshotReload::DEFAULT_AWS_CREDS}")
+
+  on("-k", "--aws-key AWSKEY",
+    "Provide the AWS key for logging in. Overrides value in AWS_CONFIG and default if given",
+    "Required if --aws-secret is specified")
+
+  on("-y", "--aws-secret AWSSECRET",
+    "Provide the AWS secret key for logging in. Overrides value in AWS_CONFIG and default if given.",
+    "Required if --aws-key is specified.")
+
+  on("-n", "--dry-run",
+    "Don't run anything, just show what would be run")
+
+  on('-v', '--verbose',
+    "Provide lots of information about what is happening")
+
+  on('-q', '--quiet',
+    "Be completely silent. Overrides --verbose switch")
+
+  arg :config, :required, 
+    "Configuration file in same format as Rails config/database.yml file"
+
+
+  version SnapshotReload::VERSION
+
+  use_log_level_option
+
+  go!
+end

data/features/snapshot_live.feature

@@ -0,0 +1,29 @@
+#
+# +Copyright+:: (c) 2012, Novu, LLC
+# +License+:: All rights reserved. For internal and private use only.
+# +Author+:: Tamara Temple <tamara.temple@novu.com>
+#
+
+Feature: Live testing of application
+  In order to reload a snapshot into a database
+  As a developer
+  I want to run the application with real credentials
+  And and a real database configuration
+
+
+  Scenario Outline: run snapshot_reload with real credentials and configuration
+    Given a file named "<config>" exists
+    Given a file named "<creds>" exists
+    When I run `snapshot_reload --log-level debug --verbose --aws-conf ../../<creds> --source s3://<bucket>/<folder>/<file> ../../<config>`
+    Then the exit status should be 0
+    And the output should match /.*Getting <folder>.<file> from <bucket>, range bytes=\d+-\d+.*/
+    And the output should match /.*Wrote \d+ bytes to <file>.*/
+    And the output should contain "SQL file: <file>"
+    And the output should match /.*Command issued: mysqladmin.*drop.*/
+    And the output should match /.*Command issued: mysqladmin.*create.*/
+    And the output should match /.*Command issued: gunzip .* | mysql .* drop.*/
+
+  Scenarios: test data
+    | config              | creds                 | bucket     | folder     | file                   |
+    | testdata/config.yml | testdata/my-aws-cred  | tam-test-2 | db-backups | novu-2012-12-11.sql.gz |
+