active_sanitization 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 621371200d0de25b03a94a5cc486bbfd8438950f
-  data.tar.gz: 96dd6d55756ac5d0cfa26c0eda7aa24099f31e1b
+  metadata.gz: 05cb2fd58e6ef73cdebaf8a54f53f97b61471fbe
+  data.tar.gz: bbc7fdc1d6a622195e58617740beac87968ab30a
 SHA512:
-  metadata.gz: 83d4f4053ac906dd77c8c2805f1e03ad524a54205bd8484086b2ade857ad48b8a7f90b54769a6f0a304c4df49719f99a2ef5c4d7861e64f31f4a61d8c05fbac8
-  data.tar.gz: f0b10f1d68af8e9e2028007dac8f1a4ad26b2692a287b4f2c3d465998e28da66760bb82e887eac2e6337d45b4c4be6b570f338f817693dcdeacecaf635798b09
+  metadata.gz: 91ce1939251bd21312769b9f76172c8272c634a6532ff808eaeb41bcca67addd553e20dfdb709c542eaaac9c48288dba163e7821b32b600296cac0446647ad3c
+  data.tar.gz: 874c6d3d1969022aa545905acb9ff9f0ce94b10658e34078486b79d7041ef8d674d752742b0389db2e4637fd38721b3a5d09c3a7aeeb71addba3d84e710c19f8

data/CHANGELOG.md

@@ -3,3 +3,9 @@ Initial release
   - Sets up gem
   - Adds dependencies
   - Exposes rake tasks
+
+# 0.2.0
+  - Add extra logging around S3 upload
+  - Fix S3 download
+  - Stop logging DB passwords
+  - Allow multiple loggers to be used

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    active_sanitization (0.1.0)
+    active_sanitization (0.2.0)
       aws-sdk (~> 2.0.33)
 
 GEM
@@ -147,7 +147,7 @@ PLATFORMS
 DEPENDENCIES
   active_sanitization!
   activerecord (~> 4.2.1)
-  bundler (~> 1.8.3)
+  bundler (~> 1.9.1)
   byebug (~> 4.0.4)
   mysql2 (~> 0.3.18)
   pry (~> 0.10.1)

data/README.md

@@ -106,9 +106,9 @@ ActiveSanitization.configure do |config|
   # The name of your app
   config.app_name = 'super_secret_app'
 
-  # The logger that the gem should use.
+  # The loggers that the gem should use.
   #  This will default to STOUT if non is provided
-  config.logger = Rails.logger
+  config.loggers = [Rails.logger]
 
   # The path to the root of your project
   #  This is required so the database dump can be put in a tmp folder

data/active_sanitization.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.8.3"
+  spec.add_development_dependency "bundler", "~> 1.9.1"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2.0"
   spec.add_development_dependency "rails", "~> 4.2.1"

data/lib/active_sanitization.rb

@@ -15,7 +15,7 @@ module ActiveSanitization
   end
 
   class Configuration
-    attr_accessor :tables_to_sanitize, :tables_to_truncate, :tables_to_ignore, :sanitization_columns, :s3_bucket, :app_name, :aws_access_key_id, :aws_secret_access_key, :env, :active_record_connection, :db_config, :custom_sanitization, :logger, :root, :s3_bucket_region
+    attr_accessor :tables_to_sanitize, :tables_to_truncate, :tables_to_ignore, :sanitization_columns, :s3_bucket, :app_name, :aws_access_key_id, :aws_secret_access_key, :env, :active_record_connection, :db_config, :custom_sanitization, :loggers, :root, :s3_bucket_region
 
     def initialize
       @tables_to_sanitize = {}
@@ -26,7 +26,7 @@ module ActiveSanitization
       @env = ENV['RACK_ENV'] || ENV['RAILS_ENV']
       @active_record_connection = ActiveRecord::Base.connection
       @root = File.dirname(File.dirname(__FILE__))
-      @logger = Logger.new(STDOUT)
+      @loggers = [Logger.new(STDOUT)]
     end
   end
 
@@ -60,7 +60,9 @@ module ActiveSanitization
   end
 
   def self.log(output)
-    self.configuration.logger.info(output) unless self.configuration.env == 'test'
+    self.configuration.loggers.each do |logger|
+      logger.info(output)
+    end unless self.configuration.env == 'test'
   end
 
   def self.pre_sanitization_checks
@@ -106,7 +108,7 @@ module ActiveSanitization
       raise "Failed to load DB #{self.configuration.db_config} into temp DB #{temp_db}."
     end
 
-    self.log("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} --no-data #{self.configuration.db_config['database']} #{self.configuration.tables_to_truncate.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
+    self.log("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=XXXXXXXXX --no-data #{self.configuration.db_config['database']} #{self.configuration.tables_to_truncate.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=XXXXXXXXX -D #{temp_db}")
     system("mysqldump -h #{self.configuration.db_config['host']} -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} --no-data #{self.configuration.db_config['database']} #{self.configuration.tables_to_truncate.keys.join(' ')} | mysql -h #{self.configuration.db_config['host']}  -u #{self.configuration.db_config['username']} --password=#{self.configuration.db_config['password']} -D #{temp_db}")
     if $?.exitstatus == 0
       self.log("Temp DB created and populated")
@@ -169,16 +171,20 @@ module ActiveSanitization
     system("gzip '#{dump_file}'")
   end
 
-  def self.get_s3_bucket
+  def self.get_s3_client
     creds = Aws::Credentials.new(self.configuration.aws_access_key_id, self.configuration.aws_secret_access_key)
-    client = Aws::S3::Client.new(credentials: creds, region: self.configuration.s3_bucket_region)
-    resource = Aws::S3::Resource.new(client: client)
+    Aws::S3::Client.new(credentials: creds, region: self.configuration.s3_bucket_region)
+  end
+
+  def self.get_s3_bucket
+    resource = Aws::S3::Resource.new(client: get_s3_client)
     resource.bucket(self.configuration.s3_bucket)
   end
 
   def self.upload(compressed_dump_file)
     timestamp = DateTime.now.strftime('%Y%m%d%H%M%S')
     name = "#{self.configuration.app_name}/#{self.configuration.env}/mysql/#{timestamp}/#{File.basename(compressed_dump_file)}"
+    self.log("Uploading to bucket: #{self.configuration.s3_bucket}, path: #{name}")
     file = File.open(compressed_dump_file, 'r')
 
     bucket = get_s3_bucket
@@ -232,9 +238,15 @@ module ActiveSanitization
 
         self.clean_up_temp_db(temp_db)
       end
+
       self.gzip(dump_file)
-      self.upload(compressed_dump_file) if self.configuration.s3_bucket && self.configuration.aws_access_key_id && self.configuration.aws_secret_access_key
-      self.clean_up_files(dump_file, compressed_dump_file) unless self.configuration.s3_bucket && self.configuration.aws_access_key_id && self.configuration.aws_secret_access_key
+
+      if self.configuration.s3_bucket && self.configuration.aws_access_key_id && self.configuration.aws_secret_access_key
+        self.upload(compressed_dump_file)
+      else
+        self.clean_up_files(dump_file, compressed_dump_file)
+      end
+
       self.log("-- DONE --")
     else
       self.log(checks[:error])
@@ -256,7 +268,7 @@ module ActiveSanitization
       return
     end
 
-    self.log('WARNING: this rake task will dump your MySQL DB to tmp, then wipe your DB before importing a snapshot')
+    self.log('WARNING: this rake task will dump your MySQL DB to a file, then wipe your DB before importing a snapshot')
     local_dump_file = "#{File.join(self.configuration.root, "tmp")}/local_data.dump"
 
     # Make copy of local DB just in case something goes wrong
@@ -267,13 +279,12 @@ module ActiveSanitization
       raise "Failed to create a local DB dump. If a previous local dump exists, please delete it and try again."
     end
 
-    # get all the files in the snapshot
-    objects = bucket.objects("#{prefix}/#{timestamp}")
     dump_file = "#{File.join(self.configuration.root, "tmp")}/data.dump"
     compressed_dump_file = "#{dump_file}.gz"
-    self.log("Downloading file to #{compressed_dump_file}")
-    url = objects.first.object.presigned_url(:get, expires_in: 600)
-    system("curl -o #{compressed_dump_file} '#{url}'")
+
+    name =  "#{prefix}/#{timestamp}/data.dump.gz"
+    self.log("Downloading dump from bucket: #{self.configuration.s3_bucket}, path: #{name}")
+    get_s3_client.get_object({ bucket:self.configuration.s3_bucket , key: name }, target: compressed_dump_file)
 
     # reset db
     self.log("Recreating your local DB")
@@ -282,7 +293,8 @@ module ActiveSanitization
 
     # Import data
     self.log("Unzipping and importing data...")
-    system("gunzip -c '#{compressed_dump_file}' | mysql -uroot #{self.configuration.db_config['database']}")
+    self.log("gunzip < #{compressed_dump_file} | mysql -u root #{self.configuration.db_config['database']}")
+    system("gunzip < #{compressed_dump_file} | mysql -u root #{self.configuration.db_config['database']}")
     if $?.exitstatus == 0
       File.delete(compressed_dump_file) if File.exist?(compressed_dump_file)
     else

data/lib/active_sanitization/version.rb

@@ -1,3 +1,3 @@
 module ActiveSanitization
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_sanitization
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Stephen Haley
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-04-03 00:00:00.000000000 Z
+date: 2015-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.9.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.9.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement