brillo 2.0.0.rc1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be0a2111b48be481799d0bd12cdf21d26910df68bbdb7606b2fb7dd60f506b4a
-  data.tar.gz: 7eb16335d290df5402ed913fd3481271da3d3f17b2f8531c8ad1f06aed143ba1
+  metadata.gz: 9c97a829dd9d79a4352fe26a35045a2a2b6a04765b12bd984ceaa9c91677edc3
+  data.tar.gz: b48b60da36cfe1f8831edcddbffbdcc98625ed27691714abe1467d84ef6d78c6
 SHA512:
-  metadata.gz: 7ee6a06c79a6db89bc4b7da52638f82335abc9edef92bc5e0cf139fb6adf380d38466c51ebe3fd4b05b43a2ce2355bbf3e2f10a0fd5492bf63f6ce32c96a2a86
-  data.tar.gz: 87a2a7018c6a494d7e3b5f1f4582dcb79b6e4f1ed77a786739bae26471f7c5d57b3caabaffa25d77d8972ea25b6b7ad3b2772cb01d71b3350ebe50f2a5289808
+  metadata.gz: 6641ac2db5299a07ff8323572b10fade253fc398710cb931cb46ea777a0bc0077d79054f7d4920b231bdb5c1019f5a0bddbfb5f69193f2cacb48ea38266e1357
+  data.tar.gz: 911695fa62235e3e6780c40cd3a9d8188cd010c1ade79b135c9c67df997dcd393eea21733b9f7c562a4a27a3b38f4336fe5734a0525fe014f9d09bf29cc09202

data/.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "monthly"

data/.github/workflows/gem-push.yml

@@ -0,0 +1,31 @@
+name: Ruby Gem
+
+on:
+  workflow_run:
+    workflows:
+      - Run Tests
+    branches:
+      - master
+    types:
+      - completed
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: ruby/setup-ruby@v1
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/run-tests.yml

@@ -0,0 +1,22 @@
+name: Run Tests
+
+on:
+  - push
+jobs:
+  test:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        gemfile: [ rails_5.1, rails_5.2, rails_6.0, rails_6.1 ]
+    env:
+      BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true
+    - name: RSpec
+      run: bundle exec appraisal ${{ matrix.gemfile }} rspec

data/.ruby-version

@@ -0,0 +1 @@
+2.6.5

data/Appraisals

@@ -1,19 +1,19 @@
-appraise "rails-4.2" do
-  gem "rails", "~> 4.2.0"
+appraise "rails_5.1" do
+  gem "rails", "~> 5.1.0"
   gem "mysql2", "~> 0.4.5"
 end
 
-appraise "rails-5.0" do
-  gem "rails", "~> 5.0.0"
+appraise "rails_5.2" do
+  gem "rails", "~> 5.2.0"
   gem "mysql2", "~> 0.4.5"
 end
 
-appraise "rails-5.1" do
-  gem "rails", "~> 5.1.0"
+appraise "rails_6.0" do
+  gem "rails", "~> 6.0.0"
   gem "mysql2", "~> 0.4.5"
 end
 
-appraise "rails-5.2" do
-  gem "rails", "~> 5.2.0"
+appraise "rails_6.1" do
+  gem "rails", "~> 6.1.0"
   gem "mysql2", "~> 0.4.5"
-end
+end

data/CHANGELOG.md

@@ -1,48 +1,80 @@
 # Change Log
 
+## Unreleased
+
+## 2.1.0
+
+- Added `recreate_db` config option which:
+  - Allows to skip dumping structures and migrations on Brillo.scrub!
+  - Allows to skip db recreation on Brillo.load!
+- Added `filename` scrub/load option to allow changing the name of the resulting scrubbed file or file to import on the DB.
+- Added config option `schema_search_path` support for Postgres so you can specify the schema (if any) on your current pg db.
+- Added `keep_local` param to Brillo.load to prevent downloading a file over and over again if the filename provided is already present. (useful for test runners)
+
+## 2.0.0
+
+Upgrade to AWS SDK 3.0, reduce dependence to S3 specifically.
+
+Support ERB in brillo.yaml
+
+Support alternative primary keys to "id"
+
+Fix bug where Brillo breaks when using the Octopus gem
+
 ## 1.3.1
+
 Disable query cacheing to reduce memory growth over time.
 
 ## 1.3.0
+
 Remove Caring specific scrubber code
 
 Fix Postgres load failing because of existing connection
 
 ## 1.2.2
+
 Fixed initializer. `Brillo.configure` pre-initialization will now work.
 
 Moved verification of config to post app initialization.
 
 ## 1.2.1
+
 Support S3 config [via your environment](https://github.com/aws/aws-sdk-ruby#configuration). To configure via the environment, just leave your `Brillo.config.transfer_config.{access_key_id,secret_access_key}` blank.
 
 ## 1.1.4
+
 Fix loading a postgres database with a password set.
 
 ## 1.1.3
+
 Fix bug in S3 upload path. Files were previously uploading to their local system path in S3. I.e. `bucket-name/my/rails/app/tmp/my-app-scrubbed.dmp.gz`.
 
 ## 1.1.2
+
 Set us-east-1 as the default region, because this is S3s default and can be accessed from other regions.
 
 ## 1.1.1
+
 Fixed postgres sequence not being set to MAX(id).
 
 ## 1.1.0
+
 **New**
+
 - **BREAKING** Brillo used to support loading your credentials from a YAML file at `/etc/ec2_secure_env.yml`
-but no longer does. It is now your responsibility to ensure the credentials are in the environment Brillo
-runs in.
+  but no longer does. It is now your responsibility to ensure the credentials are in the environment Brillo
+  runs in.
 - Removed the dependency on the AWS Timkay CLI, instead using the AWS gem.
 - Added support for configuring all S3 parameters in Ruby land
 
 **Fixed**
+
 - Fix mysql exec when no host specified
 - Fix postgres exec when no host specified
 - Fix postgres adapter reference
 
-
 ## 1.0.0
+
 First public Brillo version!
 
 ## 0.3.0

data/README.md

@@ -1,4 +1,3 @@
-[![Build Status](https://travis-ci.org/bessey/brillo.svg?branch=master)](https://travis-ci.org/bessey/brillo)
 [![Gem Version](https://badge.fury.io/rb/brillo.svg)](https://badge.fury.io/rb/brillo)
 
 # Brillo
@@ -42,28 +41,35 @@ set :brillo_role, :my_batch_role
 Here's an example `brillo.yml` for IMDB:
 
 ```yaml
-name: imdb            # Namespace the scrubbed file will occupy in S3
+name: imdb # Namespace the scrubbed file will occupy in S3
+compress: true # Compresses the file after scrubbing (default: true)
 explore:
-  user:               # Name of ActiveRecord class in snake_case
-    tactic: all       # Scrubbing tactic to use (see Brillo:TACTICS for choices)
-    associations:     # Associations to include in the scrub (ALL associated records included)
+  user: # Name of ActiveRecord class in snake_case
+    tactic: all # Scrubbing tactic to use (see Brillo:TACTICS for choices)
+    associations: # Associations to include in the scrub (ALL associated records included)
       - comments
   movie:
-    tactic: latest    # The latest tactic explores the most recent 1,000 records
+    tactic: latest # The latest tactic explores the most recent 1,000 records
     associations:
       - actors
       - ratings
-  admin/note:         # Corresponds to the Admin::Note class
+  admin/note: # Corresponds to the Admin::Note class
     tactic: all
-obfuscations:         #
-  user.name: name     # Scrub user.name with the "name" scrubber (see Brillo::SCRUBBERS for choices)
+obfuscations: #
+  user.name: name # Scrub user.name with the "name" scrubber (see Brillo::SCRUBBERS for choices)
   user.phone: phone
   user.email: email
 ```
 
 Brillo uses [the official aws-sdk](https://github.com/aws/aws-sdk-ruby) to communicate with S3. There [are a number of ways](https://github.com/aws/aws-sdk-ruby#configuration) to pass your S3 credentials, but the simplest is to set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your environment.
 
-If you'd like to see the gem in use, check out the  [/example_app](https://github.com/bessey/brillo/tree/master/example_app) directory.
+If you'd like to see the gem in use, check out the [/example_app](https://github.com/bessey/brillo/tree/master/example_app) directory.
+
+### Creating a scrubbed copy of the database in production
+
+```bash
+$ rake db:scrub
+```
 
 ### Loading a database in development
 

data/config/brillo-example.yml

@@ -1,6 +1,7 @@
 name: my-app          # Namespace the scrubbed file will occupy
 ## Optional configuration (with defaults shown)
 # compress: true      # Enables gzip on scrub and ungzip on load
+# recreate_db: true   # Dump structures and migrations on scrub and recreated database on load
 #
 # obfuscations:
 #   user.name: name   # Scrub table.field with scrubber (see Brillo::SCRUBBERS for choices)

data/gemfiles/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_RETRY: "1"

data/gemfiles/{rails_4.2.gemfile → rails_6.0.gemfile}

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "rails", "~> 4.2.0"
+gem "rails", "~> 6.0.0"
 gem "mysql2", "~> 0.4.5"
 
 gemspec path: "../"

data/gemfiles/{rails_5.0.gemfile → rails_6.1.gemfile}

@@ -2,7 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.0.0"
+gem "rails", "~> 6.1.0"
 gem "mysql2", "~> 0.4.5"
 
 gemspec path: "../"

data/lib/brillo.rb

@@ -27,20 +27,26 @@ module Brillo
     yield config
   end
 
-  def self.scrub!(logger: ::Logger.new(STDOUT))
+  def self.scrub!(logger: ::Logger.new(STDOUT), filename: nil)
     Brillo::Logger.logger = logger
-    Scrubber.new(config).scrub!
+    configuration = config
+    configuration.app_name = filename if filename
+    Scrubber.new(configuration).scrub!
   end
 
-  def self.load!(logger: ::Logger.new(STDOUT))
+  def self.load!(keep_local: false, logger: ::Logger.new(STDOUT), filename: nil)
     Brillo::Logger.logger = logger
-    Loader.new(config).load!
+    configuration = config
+    configuration.app_name = filename if filename
+    Loader.new(configuration).load! keep_local
   end
 
   def self.config
     @config ||= begin
       static_config = YAML.load(ERB.new(File.read("#{Rails.root.to_s}/config/brillo.yml")).result).deep_symbolize_keys
-      Config.new(static_config)
+      c = Config.new(static_config)
+      yield c if block_given?
+      c
     end
   end
 

data/lib/brillo/adapter/mysql.rb

@@ -25,9 +25,9 @@ module Brillo
       end
 
       def load_command
-        host = config[:host] ? "--host #{config[:host]}" : ""
-        password = config[:password] ? "-p#{config[:password]}" : ""
-        "mysql #{host} -u #{config[:username]} #{password} #{config[:database]}"
+        host = config["host"] ? "--host #{config["host"]}" : ""
+        password = config["password"] ? "-p#{config["password"]}" : ""
+        "mysql #{host} -u #{config.fetch("username")} #{password} #{config.fetch("database")}"
       end
     end
   end

data/lib/brillo/adapter/postgres.rb

@@ -2,9 +2,11 @@ module Brillo
   module Adapter
     class Postgres < Base
       def load_command
-        host = config[:host] ? "--host #{config[:host]}" : ""
-        password = config[:password] ? "PGPASSWORD=#{config[:password]} " : ""
-        "#{password}psql #{host} -U #{config[:username]} #{config[:database]}"
+        host = config["host"] ? "--host #{config["host"]}" : ""
+        password = config["password"] ? "PGPASSWORD=#{config["password"]} " : ""
+        search_path = config["schema_search_path"] ? "PGOPTIONS=--search_path=#{config["schema_search_path"]} " : ""
+        inline_options = password + search_path
+        "#{inline_options}psql #{host} -U #{config.fetch("username")} #{config.fetch("database")}"
       end
 
       # pgdump without schema does not set sequences, so we have to do it ourselves, or the first insert

data/lib/brillo/config.rb

@@ -1,11 +1,12 @@
 module Brillo
   class Config
-    attr_reader :app_name, :compress, :obfuscations, :klass_association_map, :db, :transfer_config
+    attr_accessor :app_name, :compress, :obfuscations, :klass_association_map, :db, :transfer_config, :recreate_db
 
     def initialize(options = {})
       @app_name =               options.fetch(:name)
       @klass_association_map =  options[:explore] || {}
       @compress =               options.fetch(:compress,  true)
+      @recreate_db =            options.fetch(:recreate_db,  true)
       @transfer_config =        Transferrer::Config.new(**options.fetch(:transfer, {}))
       @obfuscations =           parse_obfuscations(options[:obfuscations] || {})
     rescue KeyError => e
@@ -53,7 +54,7 @@ module Brillo
     end
 
     def db
-      @db_config ||= ActiveRecord::Base.connection.instance_variable_get(:@config).dup
+      @db_config ||= Rails.configuration.database_configuration[Rails.env].dup
     end
 
     # TODO support other transfer systems
@@ -62,13 +63,13 @@ module Brillo
     end
 
     def adapter
-      case db[:adapter].to_sym
+      case db["adapter"].to_sym
       when :mysql2
         Adapter::MySQL.new(db)
       when :postgresql
         Adapter::Postgres.new(db)
       else
-        raise ConfigParseError, "Unsupported DB adapter #{db[:adapter]}"
+        raise ConfigParseError, "Unsupported DB adapter #{db["adapter"]}"
       end
     end
 

data/lib/brillo/dumper/mysql_dumper.rb

@@ -13,11 +13,11 @@ module Brillo
         execute!(
           "mysqldump",
           host_arg,
-          "-u #{db[:username]}",
+          "-u #{db["username"]}",
           password_arg,
           "--no-data",
           "--single-transaction", # InnoDB only. Prevent MySQL locking the whole database during dump.
-          "#{db[:database]}",
+          "#{db["database"]}",
           "> #{config.dump_path}"
         )
       end
@@ -25,7 +25,7 @@ module Brillo
       private
 
       def password_arg
-        if password = config.db[:password].presence
+        if password = config.db["password"].presence
           "--password=#{password}"
         else
           ""
@@ -33,7 +33,7 @@ module Brillo
       end
 
       def host_arg
-        if (host = config.db[:host].presence) && host != 'localhost'
+        if (host = config.db["host"].presence) && host != 'localhost'
           "-h #{host}"
         else
           ""

data/lib/brillo/loader.rb

@@ -11,13 +11,23 @@ module Brillo
       @config = config
     end
 
-    def load!
-      config.transferrer.download
+    def load!(keep_local)
+      download_sql(keep_local)
       recreate_db
       import_sql
     end
 
+    def download_sql(keep_local)
+      if keep_local
+        path = config.compress ? config.compressed_dump_path : config.dump_path
+        return if File.exists? path
+      end
+
+      config.transferrer.download
+    end
+
     def recreate_db
+      return unless config.recreate_db
       config.adapter.recreate_db
     end
 

data/lib/brillo/scrubber.rb

@@ -16,7 +16,7 @@ module Brillo
       phone:        ->(n) { n = n.split(' ').first; n && n.length > 9 ? n[0..-5] + n[-1] + n[-2] + n[-3] + n[-4] : n},
       name:         ->(n) { n.downcase.split(' ').map do |word|
           word.chars.shuffle!(random: JUMBLE_PRNG.clone).join
-        end.each(&:capitalize!).join(' ')
+      end.each(&:capitalize!).join(' ')
       },
     }
 
@@ -35,12 +35,17 @@ module Brillo
     def scrub!
       FileUtils.rm config.compressed_filename, force: true
       configure_polo
-      adapter.dump_structure_and_migrations(config)
+      dump_structure_and_migrations
       explore_all_classes
       compress
       config.transferrer.upload
     end
 
+    def dump_structure_and_migrations
+      return unless config.recreate_db
+      adapter.dump_structure_and_migrations(config)
+    end
+
     def explore_all_classes
       File.open(config.dump_path, "a") do |sql_file|
         sql_file.puts(adapter.header)
@@ -93,7 +98,7 @@ module Brillo
 
     def configure_polo
       obfs = obfuscations
-      adapter = config.db[:adapter]
+      adapter = config.db["adapter"]
       Polo.configure do
         obfuscate obfs
         if adapter == "mysql2"

data/lib/brillo/version.rb

@@ -1,3 +1,3 @@
 module Brillo
-  VERSION = "2.0.0.rc1"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: brillo
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc1
+  version: 2.1.0
 platform: ruby
 authors:
 - Matt Bessey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-25 00:00:00.000000000 Z
+date: 2021-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -157,9 +157,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/gem-push.yml"
+- ".github/workflows/run-tests.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- ".ruby-version"
 - Appraisals
 - CHANGELOG.md
 - Gemfile
@@ -171,10 +174,11 @@ files:
 - brillo.gemspec
 - config/brillo-example.yml
 - config/brillo-initializer.rb
-- gemfiles/rails_4.2.gemfile
-- gemfiles/rails_5.0.gemfile
+- gemfiles/.bundle/config
 - gemfiles/rails_5.1.gemfile
 - gemfiles/rails_5.2.gemfile
+- gemfiles/rails_6.0.gemfile
+- gemfiles/rails_6.1.gemfile
 - lib/brillo.rb
 - lib/brillo/adapter/README.md
 - lib/brillo/adapter/base.rb
@@ -211,11 +215,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Rails database scrubber and loader, great for seeding your dev db with real

data/.travis.yml

@@ -1,27 +0,0 @@
-language: ruby
-
-rvm:
-  - 2.3.1
-  - 2.5.1
-
-gemfile:
-  - gemfiles/rails_4.2.gemfile
-  - gemfiles/rails_5.0.gemfile
-  - gemfiles/rails_5.1.gemfile
-  - gemfiles/rails_5.2.gemfile
-
-cache: bundler
-
-services:
-  - mysql
-
-before_install:
-  - mysql -e "create database IF NOT EXISTS brillo_test;" -uroot
-
-script:
-  - bundle exec rspec
-  - cd example_app; bundle exec rspec
-
-matrix:
-  allow_failures:
-  - gemfile: gemfiles/rails_5.2.gemfile