snapshot_reload 1.0.0

data/features/snapshot_reload.feature

@@ -0,0 +1,171 @@
+#
+# +Copyright+:: (c) 2012, Novu, LLC
+# +License+:: All rights reserverd. For internal and private use only
+# +Author+:: Tamara Temple <tamara.temple@novu.com>
+#
+
+# TODO: consolidate scenarios into outlines to make them DRYer
+
+Feature: ensure command line options work as expected
+  In order to test the command line options
+  As a tester
+  I want to try out all the options
+
+  Scenario: learn how to use application
+    When I get help for "snapshot_reload"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+      |--log-level LEVEL|
+      |--env ENVIRONMENT|
+      |--source S3SOURCE|
+      |--aws-conf AWS_CONFIG|
+      |--aws-key AWSKEY|
+      |--aws-secret AWSSECRET|
+    And the banner should document that this app's arguments are:
+      |config|which is required|
+
+  Scenario: omit a configuration file
+    When I run `snapshot_reload --log-level debug`
+    Then the output should contain "parse error: 'config' is required"
+    And the banner should be present
+    And the exit status should not be 0
+
+  Scenario: provide non-existant configuration file
+    When I run `snapshot_reload --log-level debug --dry-run config`
+    Then the exit status should be 1
+    And the output should contain "config file does not exist"
+
+  Scenario: provide non-yaml configuration file
+    Given a file named "config.txt" with:
+      """
+      Hello There!
+      """
+    When I run `snapshot_reload --dry-run config.txt`
+    Then the exit status should be 2
+    And the output should contain "config.txt is not YAML"
+  
+  Scenario: provide existing configuration file and no credential file
+    Given a file named "config.yaml" with:
+      """
+      qa:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-conf aws_cred --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 4
+    And the output should contain "aws_cred does not exist!"
+  
+  
+
+  Scenario: provide existing configuration file and credential file
+    Given a file named "config.yaml" with:
+      """
+      qa:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    Given a file named "aws_cred" with:
+      """
+      access_key = A1234
+      secret_key = S5678
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-conf aws_cred --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 0
+  
+  Scenario: provide existing configuration file with environment "other" and credential file
+    Given a file named "config.yaml" with:
+      """
+      other:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    Given a file named "aws_cred" with:
+      """
+      access_key = A1234
+      secret_key = S5678
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-conf aws_cred --env other --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 0
+
+  Scenario: provide existing configuration file with environment "other" and aws-key but omit awk-secret
+    Given a file named "config.yaml" with:
+      """
+      other:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-key A1234 --env other --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 9
+    And the output should contain "Must provide *both* aws-key and aws-secret"
+
+  Scenario: provide existing configuration file with environment "other" and aws-secret but omit awk-key
+    Given a file named "config.yaml" with:
+      """
+      other:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-secret S5678 --env other --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 9
+    And the output should contain "Must provide *both* aws-key and aws-secret"
+
+  Scenario: provide --verbose switch
+    Given a file named "config.yaml" with:
+      """
+      qa:
+        adapter: mysql2
+        host: localhost
+        encoding: utf8
+        database: novu_test
+        pool: 5
+        username: novuadmin
+        password: 
+      """
+    Given a file named "aws_cred" with:
+      """
+      access_key = A1234
+      secret_key = S5678
+      """
+    When I run `snapshot_reload --log-level debug --dry-run --aws-conf aws_cred --verbose --source s3://tam-test-2/db-backups/novu-2012-12-11.sql.gz config.yaml`
+    Then the exit status should be 0
+    And the output should contain "config: "
+    And the output should contain "env: qa"
+    And the output should contain "host: localhost"
+    And the output should contain "database: novu_test"
+    And the output should contain "username: novuadmin"
+    And the output should contain "password: "
+    And the output should contain "source: "
+    And the output should contain "aws key: A1234"
+    And the output should contain "aws secret: S5678"
+    And the output should contain "dry run: true"
+    And the output should contain "verbose: true"
+    And the output should contain "quiet: false"
+

data/features/step_definitions/snapshot_live_steps.rb

@@ -0,0 +1,4 @@
+Given /^a file named "(.*?)" exists$/ do |arg1|
+  # peending # express the regexp above with the code you wish you had
+  File.exists?(arg1)
+end

data/features/step_definitions/snapshot_reload_steps.rb

@@ -0,0 +1 @@
+# Put your step definitions here

data/features/support/env.rb

@@ -0,0 +1,17 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+  @aruba_timeout_seconds = 3600 # need enough time to transfer the large sql file
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/snapshot_reload.rb

@@ -0,0 +1,129 @@
+=begin
+
+   snapshot_reload.rb
+
+   +Copyright:+:: (c) 2012, Novu, LLC
+   +License:+::   All rights reserved. For internatl and private use only.
+   +Author:+::    Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+
+require "snapshot_reload/version"
+require "snapshot_reload/errors"
+require "snapshot_reload/defaults"
+require "snapshot_reload/validate"
+require "snapshot_reload/fetch"
+require "snapshot_reload/reload"
+require "snapshot_reload/String"
+require 'methadone'
+
+module SnapshotReload
+  class SnapshotReload
+    include Methadone::CLILogging
+
+    # attr_reader :config, :env, :host, :database, :username, :password, :source, :aws_key, :aws_secret, :sql_file
+
+    def initialize(config, options=nil)
+
+      options = Hash.new if options.nil?
+
+      @config = validate_configuration(config)
+      @env = validate_environment(options[:env], @config)
+
+      @host = check_field(@config,@env,'host')
+      @database = check_field(@config,@env,'database')
+      @username = check_field(@config,@env,'username')
+      @password = check_field(@config,@env,'password',false)
+
+      @source = validate_source(options[:source])
+
+      if @source.match('^s3://')
+        aws = validate_aws(options['aws-conf'],
+          options['aws-key'], options['aws-secret'])
+        @aws_key = aws[0]
+        @aws_secret = aws[1]
+      end
+
+      @dry_run = options['dry-run'] ||= false
+
+      @verbose = options[:verbose] ||= false
+      @quiet = options[:quiet] ||= false
+
+      @verbose = false if @quiet
+
+      @sql_file = ''
+
+      if @verbose
+        info("config: #{@config.to_s}")
+        info("env: #{@env.to_s}")
+        info("host: #{@host.to_s}")
+        info("database: #{@database.to_s}")
+        info("username: #{@username.to_s}")
+        info("password: #{@password.to_s}")
+        info("source: #{@source.to_s}")
+        info("aws key: #{@aws_key}")
+        info("aws secret: #{@aws_secret}")
+        info("dry run: #{@dry_run}")
+        info("verbose: #{@verbose}")
+        info("quiet: #{@quiet}")
+      end
+
+      reload # and here all the magic happens!!  
+      
+    end
+    
+    def config
+      @config
+    end
+
+    def env
+      @env
+    end
+
+    def host
+      @host
+    end
+
+    def database
+      @database
+    end
+
+    def username
+      @username
+    end
+
+    def password
+      @password
+    end
+
+    def source
+      @source
+    end
+
+    def aws_key
+      @aws_key
+    end
+
+    def aws_secret
+      @aws_secret
+    end
+
+    def sql_file
+      @sql_file
+    end
+
+    def dry_run?
+      @dry_run      
+    end
+
+    def verbose?
+      @verbose
+    end
+
+    def quiet?
+      @quiet
+    end
+
+  end
+end

data/lib/snapshot_reload/String.rb

@@ -0,0 +1,25 @@
+=begin
+
+   String.rb
+
+   +Copyright:+:: (c) 2012, Novu, LLC
+   +License:+::   All rights reserved. For internatl and private use only.
+   +Author:+::    Tamara Temple <tamara.temple@novu.com>
+
+   Additional methods for String and NilClass, cos I want to.
+   Seriously, doesn't "present?" look better that "not nil?" ??
+
+=end
+
+
+class String
+  def present?
+    true unless self.nil? or self.empty?
+  end
+end
+
+class NilClass
+  def present?
+    true unless self.nil?    
+  end
+end

data/lib/snapshot_reload/defaults.rb

@@ -0,0 +1,18 @@
+=begin
+
+   Defaults used in the SnapshotReload module
+
+   +Copyright:+ (c) 2012 Novu, LLC
+   +License:+ All rights reserved. For internal and private use only
+   +Author:+ Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+module SnapshotReload
+  DEFAULT_S3_SOURCE = %q{s3://novu_backups/db_backups/clean-mysqldump.sql.gz}
+  DEFAULT_AWS_CREDS = %q{/opt/novu/.s3cfg}
+  DEFAULT_ENVIRONMENT = %q{qa}
+
+  CHUNK_SIZE = 10 * 1024 * 1024 # chunk size to pull over file from S3
+end
+

data/lib/snapshot_reload/errors.rb

@@ -0,0 +1,31 @@
+=begin
+
+   errors.rb
+
+   +Copyright:+:: (c) 2012, Novu, LLC
+   +License:+::   All rights reserved. For internatl and private use only.
+   +Author:+::    Tamara Temple <tamara.temple@novu.com>
+
+   Enumerate program exit errors as descriptive constants
+
+
+=end
+
+module SnapshotReload
+
+  ENOCONFIG       = 1
+  ECONFIGNOTYAML  = 2
+  ENOENVINCONFIG  = 3
+  ENOCRED         = 4
+  ENOFIELD        = 5
+  ENOSQLFILE      = 6
+  ECMDFAILED      = 7
+  EBADS3URI       = 8
+  EMISSINGKEYORSECRET = 9
+  ENOWRITE        = 10
+  EDROPFAILED     = 11
+  ENOBUCKET       = 12
+  ENOOBJECT       = 13
+  ENOS3FILES      = 14
+
+end

data/lib/snapshot_reload/fetch.rb

@@ -0,0 +1,161 @@
+=begin
+
+   fetch.rb
+
+   +Copyright:+:: (c) 2012, Novu, LLC
+   +License:+::   All rights reserved. For internatl and private use only.
+   +Author:+::    Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+require 'methadone'
+require 'fog'
+
+module SnapshotReload
+  class SnapshotReload
+    
+    def fetch_snapshot
+      
+      if @source.match('^s3://')
+        @sql_file = s3_fetch
+      else
+        @sql_file = @source
+      end
+
+      unless File.exists?(@sql_file)
+        fatal("#{@sql_file} does not exist!")
+        exit(ENOSQLFILE)
+      end
+
+      info("SQL file: #{@sql_file}") if @verbose
+
+      @sql_file
+
+    end
+
+
+=begin
+
+   Helper methods for fetch_snapshot
+
+=end
+
+
+    def s3_fetch
+
+      matches = @source.match('^s3://([^/]+)/(.*)$')
+      if matches
+        s3_bucket_name=matches[1]
+        s3_object_name=matches[2]
+      else
+        fatal("#{@source} is not a valid s3 uri (must be s3://bucket/object)")
+        exit(EBADS3URI)
+      end
+
+
+      warn("Dry run, nothing will be fetched from S3") if @dry_run
+
+      info("Connecting to AWS S3") if @verbose
+      
+      connection = Fog::Storage.new(:provider => 'AWS',
+        :aws_access_key_id => @aws_key,
+        :aws_secret_access_key => @aws_secret) unless @dry_run
+
+
+      unless @dry_run
+        
+        buckets = connection.directories.select do |dir|
+          debug("Dir: #{dir.key}")
+          dir if dir.key == s3_bucket_name
+        end
+
+        if buckets.nil? or buckets.empty?
+          fatal("no bucket #{s3_bucket_name} found")
+          exit(ENOBUCKET)
+        end
+
+        s3_bucket = buckets.first
+        
+        info("S3 Bucket #{s3_bucket.key} found") if @verbose
+        
+        if s3_bucket.files.nil?
+          error("No files in S3 bucket #{s3_bucket_name}")
+          exit(ENOS3FILES)
+        end
+
+        files = s3_bucket.files.select do |file| 
+          debug("File: #{file.key}")
+          file if file.key == s3_object_name 
+        end
+
+        if files.nil? or files.empty?
+          fatal("no object #{s3_object_name} found")
+          exit(ENOOBJECT)
+        end
+
+        s3_object = files.first
+
+        info("S3 Object #{s3_object_name} found in #{s3_bucket_name}") if @verbose
+
+        s3_file = File.basename(s3_object.key)
+        s3_object_content_length = s3_object.content_length
+      
+      else # this is just a dry run, make up stuff
+      
+        s3_file = File.basename(s3_object_name)
+        s3_object_content_length = 0
+      
+      end
+
+      if File.exists?(s3_file) and
+        not (File.stat(s3_file).file? and File.stat(s3_file).writable?)
+        fatal("#{s3_file} is not writable!")
+        exit(ENOWRITE)
+      end
+
+      info("Writing to file #{s3_file}") if @verbose
+
+      File.open(s3_file,'w') do |file|
+
+        # Calculate number of batches for extremely large files
+
+        batches = s3_object_content_length / CHUNK_SIZE
+
+        (0..batches).each do |batch|
+          start_byte = batch * CHUNK_SIZE
+          end_byte = start_byte + CHUNK_SIZE - 1
+          contents = get_the_object(connection, s3_bucket_name, s3_object_name, start_byte, end_byte)
+          info("Writing #{contents.length} bytes to #{s3_file}") if @verbose
+          file.write(contents)
+        end
+
+        # Now get the remainder
+        start_byte = batches * CHUNK_SIZE
+        end_byte = s3_object_content_length
+        contents = get_the_object(connection, s3_bucket_name, s3_object_name, start_byte, end_byte)
+        info("Writing #{contents.length} bytes to #{s3_file}") if @verbose
+        file.write(contents)
+
+      end # File.open
+      
+      s3_file_stat = File.stat(s3_file)
+      info("Wrote #{s3_file_stat.size} bytes to #{s3_file}") if @verbose
+
+      s3_file # return the name of file written
+
+    end # method s3_fetch
+
+    def get_the_object(cnxn, bucket, name, range_start, range_end)
+
+      get_object_options = { 'Range' => "bytes=%d-%d" % [ range_start, range_end ] }
+      info("Getting #{name} from #{bucket}, range #{get_object_options['Range']}") if @verbose
+      return '' if @dry_run
+      response = cnxn.get_object(bucket,name,get_object_options)
+      debug("Response.class: #{response.class}")
+      debug("Response.status: #{response.status}")
+      response.body
+    end
+
+  end
+  
+end