brillo 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b684f9b150cde1eded3db5ae684ca02853e6ae37
-  data.tar.gz: 1a052b7afc30957c7b87b04506be442c511b65ea
+  metadata.gz: b1b61984d544d65ece11ac029344d31a7d3bc111
+  data.tar.gz: 38187ebba9e25dda6be33418813e457a292cfb6f
 SHA512:
-  metadata.gz: b1726eac1715f1fda25a19e35f77e82bdfe528378c83736ca194ff86c0a1d6bd57eb2d203d4583d8e0cef9e758209200b3d818d167e3533f13d013b1ca0f8fb9
-  data.tar.gz: 2e9857f15ea8486806208bf64954b35eec1371fceb183b27803e828baf1d6726a718f350f50f91ffe6f23e230f80270d8529820db706dfd9a4f5f7ed62055fed
+  metadata.gz: 285e40acd59b6ebd01809c99d4c13285f36b6366022df3ed781c6d0e89198c7448ecd72ba9245bd54ffbe2c08dfb93dbcecacf88d600a4a231c4062421f00b9e
+  data.tar.gz: 4665c6cf7c9a59736a577e9db36f17725ca453328aba4486a20ef66bb20eada68407b13dc847b452558fa430d64b0d1a16228eecf97f09d8d618c14972a3190b

data/.travis.yml

@@ -1,4 +1,13 @@
 language: ruby
 rvm:
-  - 2.1.5
-before_install: gem install bundler -v 1.10.6
+  - 2.3.1
+cache: bundler
+services:
+  - mysql
+
+before_install:
+  - mysql -e "create database IF NOT EXISTS brillo_test;" -uroot
+
+script:
+  - bundle exec rspec
+  - cd example_app; bundle exec rspec

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Change Log
 
+## 1.1.0
+**New**
+- **BREAKING** Brillo used to support loading your credentials from a YAML file at `/etc/ec2_secure_env.yml`
+but no longer does. It is now your responsibility to ensure the credentials are in the environment Brillo
+runs in.
+- Removed the dependency on the AWS Timkay CLI, instead using the AWS gem.
+- Added support for configuring all S3 parameters in Ruby land
+
+**Fixed**
+- Fix mysql exec when no host specified
+- Fix postgres exec when no host specified
+- Fix postgres adapter reference
+
+
 ## 1.0.0
 First public Brillo version!
 

data/README.md

@@ -1,4 +1,5 @@
-[![Build Status](https://travis-ci.com/bessey/brillo.svg?token=z16y9ppDyNfaLAvjjHbK&branch=master)](https://travis-ci.com/bessey/brillo)
+[![Build Status](https://travis-ci.org/bessey/brillo.svg?branch=master)](https://travis-ci.org/bessey/brillo)
+[![Gem Version](https://badge.fury.io/rb/brillo.svg)](https://badge.fury.io/rb/brillo)
 
 # Brillo
 
@@ -62,7 +63,9 @@ obfuscations:         #
   user.email: email
 ```
 
-In order to communicate with S3, Brillo expects `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` to be set in the environment. It uses [Tim Kay's AWS cli](http://timkay.com/aws/) to communicate with AWS.
+Brillo uses [the official aws-sdk](https://github.com/aws/aws-sdk-ruby) to communicate with S3. There [are a number of ways](https://github.com/aws/aws-sdk-ruby#configuration) to pass your S3 credentials, but the simplest is to set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your environment.
+
+If you'd like to see the gem in use, check out the  [/example_app](https://github.com/bessey/brillo/tree/master/example_app) directory.
 
 ### Loading a database in development
 
@@ -76,7 +79,7 @@ $ rake db:load
 $ cap staging db:load
 ```
 
-### Adding scrub tactics and obfuscations
+## Advanced Configuration
 
 If the built in record selection tactics aren't enough for you, or you need a custom obfuscation strategy, you can add them via the initializer. They are available in the YAML config like any other strategy.
 
@@ -94,11 +97,14 @@ Brillo.configure do |config|
   config.add_obfuscation :phone_with_id, -> (field, instance) {
     (555_000_0000 + instance.id).to_s
   }
+
+  # In addition to setting your S3 credentials via env you can set them something like this
+  config.transfer_config.secret_access_key  = Rails.application.secrets.secret_access_key
+  config.transfer_config.access_key_id      = Rails.application.secrets.access_key_id
 end
 
 ```
 
 ## To Do
 
-- Support S3 transfer via the usual AWS CLI
 - Support alternative transfer mechanisms

data/Rakefile

@@ -1 +1,9 @@
 require "bundler/gem_tasks"
+
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+end

data/brillo.gemspec

@@ -13,18 +13,21 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/bessey/brillo"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|dummy)/}) }
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|example_app)/}) }
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "rake", "~> 10.0"
-  spec.add_runtime_dependency "capistrano", "~> 3.0"
-  spec.add_runtime_dependency "polo", "~> 0.3"
+  spec.add_dependency "rake", "~> 10.0"
+  spec.add_dependency "capistrano", "~> 3.0"
+  spec.add_dependency "polo", "~> 0.3"
+  spec.add_dependency "aws-sdk", "~> 2.0"
+  spec.add_dependency "activerecord",  ">= 3.2"
+  spec.add_dependency "activesupport", ">= 3.2"
 
-  spec.add_development_dependency "rails", ">= 3.2"
-  spec.add_development_dependency "rspec", "~> 3.4"
+  spec.add_development_dependency "rails", "4.2.5.1"
+  spec.add_development_dependency "mysql2"
+  spec.add_development_dependency "rspec-rails", "~> 3.4"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "benchmark-ips"
-  spec.add_development_dependency "geminabox"
 end

data/config/brillo-example.yml

@@ -1,11 +1,13 @@
 name: my-app          # Namespace the scrubbed file will occupy
 ## Optional configuration (with defaults shown)
-# compress: true        # Enables gzip on scrub and ungzip on load
-# send_to_s3: true      # Disable to skip sending to S3, useful for debugging
-# fetch_from_s3: true   # Disable to skip fetching from S3, useful when you have an existing scrub in tmp/
+# compress: true      # Enables gzip on scrub and ungzip on load
 #
 # obfuscations:
-#   user.name: name     # Scrub table.field with scrubber (see Brillo::SCRUBBERS for choices)
+#   user.name: name   # Scrub table.field with scrubber (see Brillo::SCRUBBERS for choices)
+transfer:
+  # enabled: true     # Disable to skip transferring to and from S3, running purely locally
+  # bucket: database-scrubs
+  # region: us-west-2 # Override to change the region to somewhere nearer you
 explore:
   user:               # Name of ActiveRecord class in snake_case
     tactic: all       # Scrubbing tactic to use (see Brillo:TACTICS for choices)

data/lib/brillo/adapter/mysql.rb

@@ -25,7 +25,9 @@ module Brillo
       end
 
       def load_command
-        "mysql --host #{config[:host]} -u #{config[:username]} #{config[:password] ? "-p#{config[:password]}" : ""} #{config[:database]}"
+        host = config[:host] ? "--host #{config[:host]}" : ""
+        password = config[:password] ? "-p#{config[:password]}" : ""
+        "mysql #{host} -u #{config[:username]} #{password} #{config[:database]}"
       end
     end
   end

data/lib/brillo/adapter/postgres.rb

@@ -2,7 +2,9 @@ module Brillo
   module Adapter
     class Postgres < Base
       def load_command
-        "psql --host #{config[:host]} -U #{config[:username]} #{config[:password] ? "-W#{config[:password]}" : ""} #{config[:database]}"
+        host = config[:host] ? "--host #{config[:host]}" : ""
+        password = config[:password] ? "-W#{config[:password]}" : ""
+        "psql #{host} -U #{config[:username]} #{password} #{config[:database]}"
       end
     end
   end

data/lib/brillo/config.rb

@@ -1,18 +1,12 @@
 module Brillo
   class Config
-    AWS_KEY_PATH = '/etc/ec2_secure_env.yml'
-    S3_BUCKET = 'scrubbed_databases2'
-    attr_reader :app_name, :compress, :obfuscations, :klass_association_map, :db, :send_to_s3, :fetch_from_s3,
-      :aws_key_path, :s3_bucket
+    attr_reader :app_name, :compress, :obfuscations, :klass_association_map, :db, :transfer_config
 
     def initialize(options = {})
       @app_name =               options.fetch("name")
       @klass_association_map =  options["explore"] || {}
       @compress =               options.fetch("compress",  true)
-      @fetch_from_s3 =          options.fetch("fetch_from_s3", true)
-      @send_to_s3 =             options.fetch("send_to_s3", true)
-      @aws_key_path =           options.fetch("aws_key_path", AWS_KEY_PATH)
-      @s3_bucket =              options.fetch("s3_bucket", S3_BUCKET)
+      @transfer_config =        Transferrer::Config.new(options.fetch("transfer", {}))
       @obfuscations =           parse_obfuscations(options["obfuscations"] || {})
     rescue KeyError => e
       raise ConfigParseError, e
@@ -46,7 +40,7 @@ module Brillo
       "#{app_name}-scrubbed.dmp"
     end
 
-    def remote_filename
+    def compressed_filename
       compress ? "#{dump_filename}.gz" : dump_filename
     end
 
@@ -54,15 +48,15 @@ module Brillo
       app_tmp + dump_filename
     end
 
-    def remote_path
-      app_tmp + remote_filename
+    def compressed_dump_path
+      app_tmp + compressed_filename
     end
 
     def db
       @db_config ||= ActiveRecord::Base.connection.instance_variable_get(:@config).dup
     end
 
-    # TODO support other tranfer systems
+    # TODO support other transfer systems
     def transferrer
       Transferrer::S3.new(self)
     end
@@ -71,7 +65,7 @@ module Brillo
       case db[:adapter].to_sym
       when :mysql2
         Adapter::MySQL.new(db)
-      when :postgres
+      when :postgresql
         Adapter::Postgres.new(db)
       else
         raise ConfigParseError, "Unsupported DB adapter #{db[:adapter]}"

data/lib/brillo/helpers/exec_helper.rb

@@ -14,7 +14,7 @@ module Brillo
         if success
           [success, stdout, stderr]
         else
-          raise RuntimeError, stderr
+          raise RuntimeError, "#{stdout} #{stderr}"
         end
       end
 

data/lib/brillo/loader.rb

@@ -7,7 +7,7 @@ module Brillo
     attr_reader :config
 
     def initialize(config)
-      raise "⚠️ DON'T LOAD IN PRODUCTION! ⚠️" if Rails.env.production?
+      raise "⚠️ DON'T LOAD IN PRODUCTION! ⚠️" if production?
       @config = config
     end
 
@@ -26,7 +26,7 @@ module Brillo
 
     def import_sql
       if config.compress
-        execute!("gunzip -c #{config.remote_path} | #{sql_load_command}")
+        execute!("gunzip -c #{config.compressed_dump_path} | #{sql_load_command}")
       else
         execute!("cat #{config.dump_path} | #{sql_load_command}")
       end
@@ -35,6 +35,10 @@ module Brillo
 
     private
 
+    def production?
+      (ENV['RAILS_ENV'] || ENV['RUBY_ENV']) == 'production'
+    end
+
     def sql_load_command
       config.adapter.load_command
     end

data/lib/brillo/railtie.rb

@@ -1,10 +1,12 @@
-module Brillo
-  class Railtie < Rails::Railtie
-    rake_tasks do
-      load "tasks/brillo.rake"
-    end
-    generators do
-      require "generators/brillo.rb"
+if defined? Rails
+  module Brillo
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load "tasks/brillo.rake"
+      end
+      generators do
+        require "generators/brillo.rb"
+      end
     end
   end
 end

data/lib/brillo/scrubber.rb

@@ -32,7 +32,7 @@ module Brillo
     end
 
     def scrub!
-      FileUtils.rm [config.dump_path, config.remote_path], force: true
+      FileUtils.rm config.compressed_filename, force: true
       configure_polo
       adapter.dump_structure_and_migrations(config)
       explore_all_classes

data/lib/brillo/transferrer/config.rb

@@ -0,0 +1,24 @@
+module Brillo
+  module Transferrer
+    class Config
+      attr_accessor :bucket, :region, :enabled, :secret_access_key, :access_key_id
+      def initialize(bucket: 'database-scrubs', region: 'us-west-2', enabled: true)
+        @enabled = enabled
+        @bucket = bucket
+        @region = region
+      end
+
+      def region
+        @region || ENV['AWS_REGION']
+      end
+
+      def secret_access_key
+        @secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'] || ENV["AWS_SECRET_KEY"] || ENV["EC2_SECRET_KEY"]
+      end
+
+      def access_key_id
+        @access_key_id || ENV['AWS_ACCESS_KEY_ID'] || ENV["AWS_ACCESS_KEY"] || ENV["EC2_ACCESS_KEY"]
+      end
+    end
+  end
+end

data/lib/brillo/transferrer/s3.rb

@@ -1,65 +1,57 @@
+require 'aws-sdk'
+
 module Brillo
   module Transferrer
     class S3
       include Helpers::ExecHelper
       include Logger
-      attr_reader :credentials, :bucket, :remote_filename, :remote_path, :download_enabled, :upload_enabled
-      attr_reader :key_path
+      attr_reader :bucket, :filename, :region, :path, :enabled
 
       def initialize(config)
-        @download_enabled = config.fetch_from_s3
-        @upload_enabled   = config.send_to_s3
-        @bucket           = config.s3_bucket
-        @key_path         = config.aws_key_path
-        @remote_filename  = config.remote_filename
-        @remote_path      = config.remote_path
-        load_credentials
+        @enabled              = config.transfer_config.enabled
+        @bucket               = config.transfer_config.bucket
+        @region               = config.transfer_config.region
+        @filename             = config.compressed_filename
+        @path                 = config.compressed_dump_path
+        Aws.config.update(
+          credentials: Aws::Credentials.new(
+            config.transfer_config.access_key_id,
+            config.transfer_config.secret_access_key
+          ),
+          region:             config.transfer_config.region
+        )
       end
 
       def download
-        return unless download_enabled
-        load_credentials
-        FileUtils.rm [config.dump_path, config.remote_path], force: true
-        aws_s3 "get"
+        return unless enabled
+        FileUtils.rm path, force: true
+        client.get_object({bucket: bucket, key: path.to_s}, target: path)
+      rescue Aws::S3::Errors::NoSuchBucket
+        create_bucket
+        retry
       end
 
       def upload
-        return unless upload_enabled
-        load_credentials
-        aws_s3 "put"
+        return unless enabled
+        object = resource.bucket(bucket).object(path.to_s)
+        object.upload_file(path)
+      rescue Aws::S3::Errors::NoSuchBucket
+        create_bucket
+        retry
       end
 
       private
 
-      def load_credentials
-        if File.exist?(key_path)
-          @credentials = YAML.load_file(key_path)
-        else
-          key = ENV["AWS_SECRET_KEY"] || ENV["EC2_SECRET_KEY"]
-          unless key && key.length > 10
-            raise CredentialsError, "AWS credentials not found. Expected AWS_ACCESS_KEY and AWS_SECRET_KEY to be set!"
-          end
-          @credentials = {
-            'aws_access_key' => ENV["AWS_ACCESS_KEY"] || ENV["EC2_ACCESS_KEY"],
-            'aws_secret_key' => key
-          }
-        end
-      end
-
-      def aws_s3 api_command
-        execute!("#{aws_env} #{aws_bin} #{api_command} #{bucket}/#{remote_filename} #{remote_path}")
+      def create_bucket
+        client.create_bucket(bucket: bucket)
       end
 
-      def aws_bin
-        if File.exist?('/usr/local/bin/awstk')
-          '/usr/local/bin/awstk'
-        else
-          '/usr/local/bin/aws'
-        end
+      def client
+        Aws::S3::Client.new
       end
 
-      def aws_env
-        "EC2_ACCESS_KEY=#{credentials['aws_access_key']} EC2_SECRET_KEY=#{credentials['aws_secret_key']}"
+      def resource
+        Aws::S3::Resource.new
       end
     end
   end

data/lib/brillo/version.rb

@@ -1,3 +1,3 @@
 module Brillo
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/brillo.rb

@@ -1,3 +1,7 @@
+require 'yaml'
+require 'active_support'
+require 'active_support/core_ext/string/inflections'
+require 'active_record'
 require "brillo/version"
 
 require 'brillo/errors'
@@ -8,6 +12,7 @@ require 'brillo/adapter/base'
 require 'brillo/adapter/mysql'
 require 'brillo/adapter/postgres'
 
+require 'brillo/transferrer/config'
 require 'brillo/transferrer/s3'
 
 require 'brillo/dumper/mysql_dumper'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: brillo
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Matt Bessey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-29 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -53,13 +53,27 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.2'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -67,21 +81,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.2'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.2.5.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: 4.2.5.1
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +123,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: benchmark-ips
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: geminabox
+  name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -155,6 +197,7 @@ files:
 - lib/brillo/logger.rb
 - lib/brillo/railtie.rb
 - lib/brillo/scrubber.rb
+- lib/brillo/transferrer/config.rb
 - lib/brillo/transferrer/s3.rb
 - lib/brillo/validator.rb
 - lib/brillo/version.rb