brillo 2.0.0.rc1 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: be0a2111b48be481799d0bd12cdf21d26910df68bbdb7606b2fb7dd60f506b4a
4
- data.tar.gz: 7eb16335d290df5402ed913fd3481271da3d3f17b2f8531c8ad1f06aed143ba1
3
+ metadata.gz: 9c97a829dd9d79a4352fe26a35045a2a2b6a04765b12bd984ceaa9c91677edc3
4
+ data.tar.gz: b48b60da36cfe1f8831edcddbffbdcc98625ed27691714abe1467d84ef6d78c6
5
5
  SHA512:
6
- metadata.gz: 7ee6a06c79a6db89bc4b7da52638f82335abc9edef92bc5e0cf139fb6adf380d38466c51ebe3fd4b05b43a2ce2355bbf3e2f10a0fd5492bf63f6ce32c96a2a86
7
- data.tar.gz: 87a2a7018c6a494d7e3b5f1f4582dcb79b6e4f1ed77a786739bae26471f7c5d57b3caabaffa25d77d8972ea25b6b7ad3b2772cb01d71b3350ebe50f2a5289808
6
+ metadata.gz: 6641ac2db5299a07ff8323572b10fade253fc398710cb931cb46ea777a0bc0077d79054f7d4920b231bdb5c1019f5a0bddbfb5f69193f2cacb48ea38266e1357
7
+ data.tar.gz: 911695fa62235e3e6780c40cd3a9d8188cd010c1ade79b135c9c67df997dcd393eea21733b9f7c562a4a27a3b38f4336fe5734a0525fe014f9d09bf29cc09202
@@ -0,0 +1,7 @@
1
+ version: 2
2
+ updates:
3
+
4
+ - package-ecosystem: "bundler"
5
+ directory: "/"
6
+ schedule:
7
+ interval: "monthly"
@@ -0,0 +1,31 @@
1
+ name: Ruby Gem
2
+
3
+ on:
4
+ workflow_run:
5
+ workflows:
6
+ - Run Tests
7
+ branches:
8
+ - master
9
+ types:
10
+ - completed
11
+
12
+ jobs:
13
+ build:
14
+ name: Build + Publish
15
+ runs-on: ubuntu-latest
16
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
17
+ steps:
18
+ - uses: actions/checkout@v2
19
+ - name: Set up Ruby 2.6
20
+ uses: ruby/setup-ruby@v1
21
+
22
+ - name: Publish to RubyGems
23
+ run: |
24
+ mkdir -p $HOME/.gem
25
+ touch $HOME/.gem/credentials
26
+ chmod 0600 $HOME/.gem/credentials
27
+ printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
28
+ gem build *.gemspec
29
+ gem push *.gem
30
+ env:
31
+ GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
@@ -0,0 +1,22 @@
1
+ name: Run Tests
2
+
3
+ on:
4
+ - push
5
+ jobs:
6
+ test:
7
+ name: Run Tests
8
+ runs-on: ubuntu-latest
9
+ strategy:
10
+ fail-fast: false
11
+ matrix:
12
+ gemfile: [ rails_5.1, rails_5.2, rails_6.0, rails_6.1 ]
13
+ env:
14
+ BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - name: Set up Ruby
18
+ uses: ruby/setup-ruby@v1
19
+ with:
20
+ bundler-cache: true
21
+ - name: RSpec
22
+ run: bundle exec appraisal ${{ matrix.gemfile }} rspec
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 2.6.5
data/Appraisals CHANGED
@@ -1,19 +1,19 @@
1
- appraise "rails-4.2" do
2
- gem "rails", "~> 4.2.0"
1
+ appraise "rails_5.1" do
2
+ gem "rails", "~> 5.1.0"
3
3
  gem "mysql2", "~> 0.4.5"
4
4
  end
5
5
 
6
- appraise "rails-5.0" do
7
- gem "rails", "~> 5.0.0"
6
+ appraise "rails_5.2" do
7
+ gem "rails", "~> 5.2.0"
8
8
  gem "mysql2", "~> 0.4.5"
9
9
  end
10
10
 
11
- appraise "rails-5.1" do
12
- gem "rails", "~> 5.1.0"
11
+ appraise "rails_6.0" do
12
+ gem "rails", "~> 6.0.0"
13
13
  gem "mysql2", "~> 0.4.5"
14
14
  end
15
15
 
16
- appraise "rails-5.2" do
17
- gem "rails", "~> 5.2.0"
16
+ appraise "rails_6.1" do
17
+ gem "rails", "~> 6.1.0"
18
18
  gem "mysql2", "~> 0.4.5"
19
- end
19
+ end
data/CHANGELOG.md CHANGED
@@ -1,48 +1,80 @@
1
1
  # Change Log
2
2
 
3
+ ## Unreleased
4
+
5
+ ## 2.1.0
6
+
7
+ - Added `recreate_db` config option which:
8
+ - Allows to skip dumping structures and migrations on Brillo.scrub!
9
+ - Allows to skip db recreation on Brillo.load!
10
+ - Added `filename` scrub/load option to allow changing the name of the resulting scrubbed file or file to import on the DB.
11
+ - Added config option `schema_search_path` support for Postgres so you can specify the schema (if any) on your current pg db.
12
+ - Added `keep_local` param to Brillo.load to prevent downloading a file over and over again if the filename provided is already present. (useful for test runners)
13
+
14
+ ## 2.0.0
15
+
16
+ Upgrade to AWS SDK 3.0, reduce dependence to S3 specifically.
17
+
18
+ Support ERB in brillo.yaml
19
+
20
+ Support alternative primary keys to "id"
21
+
22
+ Fix bug where Brillo breaks when using the Octopus gem
23
+
3
24
  ## 1.3.1
25
+
4
26
  Disable query cacheing to reduce memory growth over time.
5
27
 
6
28
  ## 1.3.0
29
+
7
30
  Remove Caring specific scrubber code
8
31
 
9
32
  Fix Postgres load failing because of existing connection
10
33
 
11
34
  ## 1.2.2
35
+
12
36
  Fixed initializer. `Brillo.configure` pre-initialization will now work.
13
37
 
14
38
  Moved verification of config to post app initialization.
15
39
 
16
40
  ## 1.2.1
41
+
17
42
  Support S3 config [via your environment](https://github.com/aws/aws-sdk-ruby#configuration). To configure via the environment, just leave your `Brillo.config.transfer_config.{access_key_id,secret_access_key}` blank.
18
43
 
19
44
  ## 1.1.4
45
+
20
46
  Fix loading a postgres database with a password set.
21
47
 
22
48
  ## 1.1.3
49
+
23
50
  Fix bug in S3 upload path. Files were previously uploading to their local system path in S3. I.e. `bucket-name/my/rails/app/tmp/my-app-scrubbed.dmp.gz`.
24
51
 
25
52
  ## 1.1.2
53
+
26
54
  Set us-east-1 as the default region, because this is S3s default and can be accessed from other regions.
27
55
 
28
56
  ## 1.1.1
57
+
29
58
  Fixed postgres sequence not being set to MAX(id).
30
59
 
31
60
  ## 1.1.0
61
+
32
62
  **New**
63
+
33
64
  - **BREAKING** Brillo used to support loading your credentials from a YAML file at `/etc/ec2_secure_env.yml`
34
- but no longer does. It is now your responsibility to ensure the credentials are in the environment Brillo
35
- runs in.
65
+ but no longer does. It is now your responsibility to ensure the credentials are in the environment Brillo
66
+ runs in.
36
67
  - Removed the dependency on the AWS Timkay CLI, instead using the AWS gem.
37
68
  - Added support for configuring all S3 parameters in Ruby land
38
69
 
39
70
  **Fixed**
71
+
40
72
  - Fix mysql exec when no host specified
41
73
  - Fix postgres exec when no host specified
42
74
  - Fix postgres adapter reference
43
75
 
44
-
45
76
  ## 1.0.0
77
+
46
78
  First public Brillo version!
47
79
 
48
80
  ## 0.3.0
data/README.md CHANGED
@@ -1,4 +1,3 @@
1
- [![Build Status](https://travis-ci.org/bessey/brillo.svg?branch=master)](https://travis-ci.org/bessey/brillo)
2
1
  [![Gem Version](https://badge.fury.io/rb/brillo.svg)](https://badge.fury.io/rb/brillo)
3
2
 
4
3
  # Brillo
@@ -42,28 +41,35 @@ set :brillo_role, :my_batch_role
42
41
  Here's an example `brillo.yml` for IMDB:
43
42
 
44
43
  ```yaml
45
- name: imdb # Namespace the scrubbed file will occupy in S3
44
+ name: imdb # Namespace the scrubbed file will occupy in S3
45
+ compress: true # Compresses the file after scrubbing (default: true)
46
46
  explore:
47
- user: # Name of ActiveRecord class in snake_case
48
- tactic: all # Scrubbing tactic to use (see Brillo:TACTICS for choices)
49
- associations: # Associations to include in the scrub (ALL associated records included)
47
+ user: # Name of ActiveRecord class in snake_case
48
+ tactic: all # Scrubbing tactic to use (see Brillo:TACTICS for choices)
49
+ associations: # Associations to include in the scrub (ALL associated records included)
50
50
  - comments
51
51
  movie:
52
- tactic: latest # The latest tactic explores the most recent 1,000 records
52
+ tactic: latest # The latest tactic explores the most recent 1,000 records
53
53
  associations:
54
54
  - actors
55
55
  - ratings
56
- admin/note: # Corresponds to the Admin::Note class
56
+ admin/note: # Corresponds to the Admin::Note class
57
57
  tactic: all
58
- obfuscations: #
59
- user.name: name # Scrub user.name with the "name" scrubber (see Brillo::SCRUBBERS for choices)
58
+ obfuscations: #
59
+ user.name: name # Scrub user.name with the "name" scrubber (see Brillo::SCRUBBERS for choices)
60
60
  user.phone: phone
61
61
  user.email: email
62
62
  ```
63
63
 
64
64
  Brillo uses [the official aws-sdk](https://github.com/aws/aws-sdk-ruby) to communicate with S3. There [are a number of ways](https://github.com/aws/aws-sdk-ruby#configuration) to pass your S3 credentials, but the simplest is to set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your environment.
65
65
 
66
- If you'd like to see the gem in use, check out the [/example_app](https://github.com/bessey/brillo/tree/master/example_app) directory.
66
+ If you'd like to see the gem in use, check out the [/example_app](https://github.com/bessey/brillo/tree/master/example_app) directory.
67
+
68
+ ### Creating a scrubbed copy of the database in production
69
+
70
+ ```bash
71
+ $ rake db:scrub
72
+ ```
67
73
 
68
74
  ### Loading a database in development
69
75
 
@@ -1,6 +1,7 @@
1
1
  name: my-app # Namespace the scrubbed file will occupy
2
2
  ## Optional configuration (with defaults shown)
3
3
  # compress: true # Enables gzip on scrub and ungzip on load
4
+ # recreate_db: true # Dump structures and migrations on scrub and recreated database on load
4
5
  #
5
6
  # obfuscations:
6
7
  # user.name: name # Scrub table.field with scrubber (see Brillo::SCRUBBERS for choices)
@@ -0,0 +1,2 @@
1
+ ---
2
+ BUNDLE_RETRY: "1"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "~> 4.2.0"
5
+ gem "rails", "~> 6.0.0"
6
6
  gem "mysql2", "~> 0.4.5"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "~> 5.0.0"
5
+ gem "rails", "~> 6.1.0"
6
6
  gem "mysql2", "~> 0.4.5"
7
7
 
8
8
  gemspec path: "../"
data/lib/brillo.rb CHANGED
@@ -27,20 +27,26 @@ module Brillo
27
27
  yield config
28
28
  end
29
29
 
30
- def self.scrub!(logger: ::Logger.new(STDOUT))
30
+ def self.scrub!(logger: ::Logger.new(STDOUT), filename: nil)
31
31
  Brillo::Logger.logger = logger
32
- Scrubber.new(config).scrub!
32
+ configuration = config
33
+ configuration.app_name = filename if filename
34
+ Scrubber.new(configuration).scrub!
33
35
  end
34
36
 
35
- def self.load!(logger: ::Logger.new(STDOUT))
37
+ def self.load!(keep_local: false, logger: ::Logger.new(STDOUT), filename: nil)
36
38
  Brillo::Logger.logger = logger
37
- Loader.new(config).load!
39
+ configuration = config
40
+ configuration.app_name = filename if filename
41
+ Loader.new(configuration).load! keep_local
38
42
  end
39
43
 
40
44
  def self.config
41
45
  @config ||= begin
42
46
  static_config = YAML.load(ERB.new(File.read("#{Rails.root.to_s}/config/brillo.yml")).result).deep_symbolize_keys
43
- Config.new(static_config)
47
+ c = Config.new(static_config)
48
+ yield c if block_given?
49
+ c
44
50
  end
45
51
  end
46
52
 
@@ -25,9 +25,9 @@ module Brillo
25
25
  end
26
26
 
27
27
  def load_command
28
- host = config[:host] ? "--host #{config[:host]}" : ""
29
- password = config[:password] ? "-p#{config[:password]}" : ""
30
- "mysql #{host} -u #{config[:username]} #{password} #{config[:database]}"
28
+ host = config["host"] ? "--host #{config["host"]}" : ""
29
+ password = config["password"] ? "-p#{config["password"]}" : ""
30
+ "mysql #{host} -u #{config.fetch("username")} #{password} #{config.fetch("database")}"
31
31
  end
32
32
  end
33
33
  end
@@ -2,9 +2,11 @@ module Brillo
2
2
  module Adapter
3
3
  class Postgres < Base
4
4
  def load_command
5
- host = config[:host] ? "--host #{config[:host]}" : ""
6
- password = config[:password] ? "PGPASSWORD=#{config[:password]} " : ""
7
- "#{password}psql #{host} -U #{config[:username]} #{config[:database]}"
5
+ host = config["host"] ? "--host #{config["host"]}" : ""
6
+ password = config["password"] ? "PGPASSWORD=#{config["password"]} " : ""
7
+ search_path = config["schema_search_path"] ? "PGOPTIONS=--search_path=#{config["schema_search_path"]} " : ""
8
+ inline_options = password + search_path
9
+ "#{inline_options}psql #{host} -U #{config.fetch("username")} #{config.fetch("database")}"
8
10
  end
9
11
 
10
12
  # pgdump without schema does not set sequences, so we have to do it ourselves, or the first insert
data/lib/brillo/config.rb CHANGED
@@ -1,11 +1,12 @@
1
1
  module Brillo
2
2
  class Config
3
- attr_reader :app_name, :compress, :obfuscations, :klass_association_map, :db, :transfer_config
3
+ attr_accessor :app_name, :compress, :obfuscations, :klass_association_map, :db, :transfer_config, :recreate_db
4
4
 
5
5
  def initialize(options = {})
6
6
  @app_name = options.fetch(:name)
7
7
  @klass_association_map = options[:explore] || {}
8
8
  @compress = options.fetch(:compress, true)
9
+ @recreate_db = options.fetch(:recreate_db, true)
9
10
  @transfer_config = Transferrer::Config.new(**options.fetch(:transfer, {}))
10
11
  @obfuscations = parse_obfuscations(options[:obfuscations] || {})
11
12
  rescue KeyError => e
@@ -53,7 +54,7 @@ module Brillo
53
54
  end
54
55
 
55
56
  def db
56
- @db_config ||= ActiveRecord::Base.connection.instance_variable_get(:@config).dup
57
+ @db_config ||= Rails.configuration.database_configuration[Rails.env].dup
57
58
  end
58
59
 
59
60
  # TODO support other transfer systems
@@ -62,13 +63,13 @@ module Brillo
62
63
  end
63
64
 
64
65
  def adapter
65
- case db[:adapter].to_sym
66
+ case db["adapter"].to_sym
66
67
  when :mysql2
67
68
  Adapter::MySQL.new(db)
68
69
  when :postgresql
69
70
  Adapter::Postgres.new(db)
70
71
  else
71
- raise ConfigParseError, "Unsupported DB adapter #{db[:adapter]}"
72
+ raise ConfigParseError, "Unsupported DB adapter #{db["adapter"]}"
72
73
  end
73
74
  end
74
75
 
@@ -13,11 +13,11 @@ module Brillo
13
13
  execute!(
14
14
  "mysqldump",
15
15
  host_arg,
16
- "-u #{db[:username]}",
16
+ "-u #{db["username"]}",
17
17
  password_arg,
18
18
  "--no-data",
19
19
  "--single-transaction", # InnoDB only. Prevent MySQL locking the whole database during dump.
20
- "#{db[:database]}",
20
+ "#{db["database"]}",
21
21
  "> #{config.dump_path}"
22
22
  )
23
23
  end
@@ -25,7 +25,7 @@ module Brillo
25
25
  private
26
26
 
27
27
  def password_arg
28
- if password = config.db[:password].presence
28
+ if password = config.db["password"].presence
29
29
  "--password=#{password}"
30
30
  else
31
31
  ""
@@ -33,7 +33,7 @@ module Brillo
33
33
  end
34
34
 
35
35
  def host_arg
36
- if (host = config.db[:host].presence) && host != 'localhost'
36
+ if (host = config.db["host"].presence) && host != 'localhost'
37
37
  "-h #{host}"
38
38
  else
39
39
  ""
data/lib/brillo/loader.rb CHANGED
@@ -11,13 +11,23 @@ module Brillo
11
11
  @config = config
12
12
  end
13
13
 
14
- def load!
15
- config.transferrer.download
14
+ def load!(keep_local)
15
+ download_sql(keep_local)
16
16
  recreate_db
17
17
  import_sql
18
18
  end
19
19
 
20
+ def download_sql(keep_local)
21
+ if keep_local
22
+ path = config.compress ? config.compressed_dump_path : config.dump_path
23
+ return if File.exists? path
24
+ end
25
+
26
+ config.transferrer.download
27
+ end
28
+
20
29
  def recreate_db
30
+ return unless config.recreate_db
21
31
  config.adapter.recreate_db
22
32
  end
23
33
 
@@ -16,7 +16,7 @@ module Brillo
16
16
  phone: ->(n) { n = n.split(' ').first; n && n.length > 9 ? n[0..-5] + n[-1] + n[-2] + n[-3] + n[-4] : n},
17
17
  name: ->(n) { n.downcase.split(' ').map do |word|
18
18
  word.chars.shuffle!(random: JUMBLE_PRNG.clone).join
19
- end.each(&:capitalize!).join(' ')
19
+ end.each(&:capitalize!).join(' ')
20
20
  },
21
21
  }
22
22
 
@@ -35,12 +35,17 @@ module Brillo
35
35
  def scrub!
36
36
  FileUtils.rm config.compressed_filename, force: true
37
37
  configure_polo
38
- adapter.dump_structure_and_migrations(config)
38
+ dump_structure_and_migrations
39
39
  explore_all_classes
40
40
  compress
41
41
  config.transferrer.upload
42
42
  end
43
43
 
44
+ def dump_structure_and_migrations
45
+ return unless config.recreate_db
46
+ adapter.dump_structure_and_migrations(config)
47
+ end
48
+
44
49
  def explore_all_classes
45
50
  File.open(config.dump_path, "a") do |sql_file|
46
51
  sql_file.puts(adapter.header)
@@ -93,7 +98,7 @@ module Brillo
93
98
 
94
99
  def configure_polo
95
100
  obfs = obfuscations
96
- adapter = config.db[:adapter]
101
+ adapter = config.db["adapter"]
97
102
  Polo.configure do
98
103
  obfuscate obfs
99
104
  if adapter == "mysql2"
@@ -1,3 +1,3 @@
1
1
  module Brillo
2
- VERSION = "2.0.0.rc1"
2
+ VERSION = "2.1.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: brillo
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.rc1
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matt Bessey
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-02-25 00:00:00.000000000 Z
11
+ date: 2021-04-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -157,9 +157,12 @@ executables: []
157
157
  extensions: []
158
158
  extra_rdoc_files: []
159
159
  files:
160
+ - ".github/dependabot.yml"
161
+ - ".github/workflows/gem-push.yml"
162
+ - ".github/workflows/run-tests.yml"
160
163
  - ".gitignore"
161
164
  - ".rspec"
162
- - ".travis.yml"
165
+ - ".ruby-version"
163
166
  - Appraisals
164
167
  - CHANGELOG.md
165
168
  - Gemfile
@@ -171,10 +174,11 @@ files:
171
174
  - brillo.gemspec
172
175
  - config/brillo-example.yml
173
176
  - config/brillo-initializer.rb
174
- - gemfiles/rails_4.2.gemfile
175
- - gemfiles/rails_5.0.gemfile
177
+ - gemfiles/.bundle/config
176
178
  - gemfiles/rails_5.1.gemfile
177
179
  - gemfiles/rails_5.2.gemfile
180
+ - gemfiles/rails_6.0.gemfile
181
+ - gemfiles/rails_6.1.gemfile
178
182
  - lib/brillo.rb
179
183
  - lib/brillo/adapter/README.md
180
184
  - lib/brillo/adapter/base.rb
@@ -211,11 +215,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
211
215
  version: '0'
212
216
  required_rubygems_version: !ruby/object:Gem::Requirement
213
217
  requirements:
214
- - - ">"
218
+ - - ">="
215
219
  - !ruby/object:Gem::Version
216
- version: 1.3.1
220
+ version: '0'
217
221
  requirements: []
218
- rubygems_version: 3.0.1
222
+ rubygems_version: 3.0.3
219
223
  signing_key:
220
224
  specification_version: 4
221
225
  summary: Rails database scrubber and loader, great for seeding your dev db with real
data/.travis.yml DELETED
@@ -1,27 +0,0 @@
1
- language: ruby
2
-
3
- rvm:
4
- - 2.3.1
5
- - 2.5.1
6
-
7
- gemfile:
8
- - gemfiles/rails_4.2.gemfile
9
- - gemfiles/rails_5.0.gemfile
10
- - gemfiles/rails_5.1.gemfile
11
- - gemfiles/rails_5.2.gemfile
12
-
13
- cache: bundler
14
-
15
- services:
16
- - mysql
17
-
18
- before_install:
19
- - mysql -e "create database IF NOT EXISTS brillo_test;" -uroot
20
-
21
- script:
22
- - bundle exec rspec
23
- - cd example_app; bundle exec rspec
24
-
25
- matrix:
26
- allow_failures:
27
- - gemfile: gemfiles/rails_5.2.gemfile