pii_safe_schema 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +118 -0
- data/.github/CODEOWNERS +1 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.gitignore +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +3 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +62 -0
- data/Gemfile +3 -0
- data/Guardfile +35 -0
- data/README.md +46 -0
- data/Rakefile +8 -0
- data/lib/pii_safe_schema/annotations.rb +87 -0
- data/lib/pii_safe_schema/configuration.rb +59 -0
- data/lib/pii_safe_schema/migration_generator.rb +53 -0
- data/lib/pii_safe_schema/notifiers/data_dog.rb +31 -0
- data/lib/pii_safe_schema/notifiers/std_out.rb +30 -0
- data/lib/pii_safe_schema/notify.rb +19 -0
- data/lib/pii_safe_schema/pii_column.rb +45 -0
- data/lib/pii_safe_schema/railtie.rb +9 -0
- data/lib/pii_safe_schema/version.rb +3 -0
- data/lib/pii_safe_schema.rb +38 -0
- data/lib/tasks/pii_safe_schema.rake +5 -0
- data/pii-safe-schema.gemspec +43 -0
- metadata +318 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 8bca88cff66431d4ed83b6f8e3956e40686d858b47beac08a0efe05eae5a8b42
|
4
|
+
data.tar.gz: 8ab4b451afbf182da050c29e6e4f8d9ea00cdd225940604b242530db6f8d3649
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5e2ddf4cbe30dbe6285581508e4d138de72acf42f031045879cf85f74a28501970f4bb7a3af3dc612e1d4eef248d94154c8c8149e6d3af79ac63b2766fdb24e6
|
7
|
+
data.tar.gz: 625c1edd58177ce1f79fada40a3e25bd1701133516ef73579b40496e81e4cdbaa97368f74cbec1d7098b0e8c23969d264cfc2b16ac140dbaa61a03b840b712cc
|
@@ -0,0 +1,118 @@
|
|
1
|
+
version: 2
|
2
|
+
|
3
|
+
defaults: &defaults
|
4
|
+
working_directory: /home/circleci/wealthsimple
|
5
|
+
docker:
|
6
|
+
- image: circleci/ruby:2.6.0
|
7
|
+
- image: circleci/postgres:9.5.9-alpine
|
8
|
+
environment:
|
9
|
+
POSTGRES_USER: circleci
|
10
|
+
POSTGRES_DB: pii_safe_schema_test
|
11
|
+
|
12
|
+
# These are common snippets that are referenced in multiple workflows.
|
13
|
+
references:
|
14
|
+
attach_code_workspace: &attach_code_workspace
|
15
|
+
attach_workspace:
|
16
|
+
at: /home/circleci/wealthsimple
|
17
|
+
|
18
|
+
restore_bundle_dependencies: &restore_bundle_dependencies
|
19
|
+
run:
|
20
|
+
name: Restore bundle dependencies from workspace
|
21
|
+
command: bundle --path vendor/bundle
|
22
|
+
|
23
|
+
jobs:
|
24
|
+
checkout_and_bundle:
|
25
|
+
<<: *defaults
|
26
|
+
steps:
|
27
|
+
- checkout
|
28
|
+
- run:
|
29
|
+
command: bundle install --jobs=4 --retry=3 --path vendor/bundle
|
30
|
+
- persist_to_workspace:
|
31
|
+
root: .
|
32
|
+
paths: .
|
33
|
+
|
34
|
+
rspec:
|
35
|
+
<<: *defaults
|
36
|
+
steps:
|
37
|
+
- *attach_code_workspace
|
38
|
+
- *restore_bundle_dependencies
|
39
|
+
- run:
|
40
|
+
command: sudo apt install -y postgresql-client || true
|
41
|
+
- run:
|
42
|
+
command: bundle exec bundle-audit update && bundle exec bundle-audit check
|
43
|
+
- run:
|
44
|
+
command: bundle exec rspec
|
45
|
+
|
46
|
+
lint_check:
|
47
|
+
<<: *defaults
|
48
|
+
steps:
|
49
|
+
- *attach_code_workspace
|
50
|
+
- *restore_bundle_dependencies
|
51
|
+
- run:
|
52
|
+
command: bundle exec rubocop
|
53
|
+
|
54
|
+
vulnerability_check:
|
55
|
+
<<: *defaults
|
56
|
+
steps:
|
57
|
+
- *attach_code_workspace
|
58
|
+
- *restore_bundle_dependencies
|
59
|
+
- run:
|
60
|
+
command: bundle exec bundle-audit update && bundle exec bundle-audit check
|
61
|
+
|
62
|
+
release:
|
63
|
+
<<: *defaults
|
64
|
+
steps:
|
65
|
+
- add_ssh_keys:
|
66
|
+
fingerprints:
|
67
|
+
- "46:b5:cb:ee:57:dc:14:95:31:be:12:13:4f:11:94:a4"
|
68
|
+
- *attach_code_workspace
|
69
|
+
- *restore_bundle_dependencies
|
70
|
+
- run:
|
71
|
+
name: Release to rubygems.org
|
72
|
+
command: |
|
73
|
+
mkdir ~/.gem
|
74
|
+
echo ":rubygems_api_key: ${RUBYGEMS_API_KEY}" >> ~/.gem/credentials
|
75
|
+
chmod 600 ~/.gem/credentials
|
76
|
+
mkdir -p ~/.ssh
|
77
|
+
echo "github.com,192.30.253.112 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts
|
78
|
+
bundle exec rake release
|
79
|
+
|
80
|
+
workflows:
|
81
|
+
version: 2
|
82
|
+
build_and_test:
|
83
|
+
jobs:
|
84
|
+
- checkout_and_bundle:
|
85
|
+
context: wealthsimple
|
86
|
+
- rspec:
|
87
|
+
requires:
|
88
|
+
- checkout_and_bundle
|
89
|
+
- lint_check:
|
90
|
+
requires:
|
91
|
+
- checkout_and_bundle
|
92
|
+
- vulnerability_check:
|
93
|
+
requires:
|
94
|
+
- checkout_and_bundle
|
95
|
+
- release:
|
96
|
+
context: wealthsimple
|
97
|
+
filters:
|
98
|
+
branches:
|
99
|
+
only: master
|
100
|
+
requires:
|
101
|
+
- rspec
|
102
|
+
- lint_check
|
103
|
+
- vulnerability_check
|
104
|
+
|
105
|
+
security-audit:
|
106
|
+
triggers:
|
107
|
+
- schedule:
|
108
|
+
# 11:45 am UTC: 6:45 am EST / 7:45 am EDT
|
109
|
+
cron: "45 11 * * *"
|
110
|
+
filters:
|
111
|
+
branches:
|
112
|
+
only: master
|
113
|
+
jobs:
|
114
|
+
- checkout_and_bundle:
|
115
|
+
context: wealthsimple
|
116
|
+
- vulnerability_check:
|
117
|
+
requires:
|
118
|
+
- checkout_and_bundle
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @wealthsimple/security
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#### Why <!-- A short description of why this change is required -->
|
2
|
+
|
3
|
+
|
4
|
+
|
5
|
+
#### What changed <!-- Summary of changes when modifying hundreds of lines -->
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
<!--
|
10
|
+
Consider adding the following sections:
|
11
|
+
|
12
|
+
#### How I tested [ Bullets for test cases covered ]
|
13
|
+
#### Next steps [ If your PR is part of a few or a WIP, give context to reviewers ]
|
14
|
+
#### Screenshot [ An image is worth a thousand words ]
|
15
|
+
#### Bug/Ticket tracker [ Unnecessary when prefixing branch with JIRA ticket, e.g. SECURITY-123-human-readable-thing ]
|
16
|
+
-->
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.6.0
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
# Changelog
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
|
+
|
7
|
+
## 1.0.3
|
8
|
+
### Fixed
|
9
|
+
- bumping version to release to rubygems
|
10
|
+
|
11
|
+
## 1.0.2
|
12
|
+
### Fixed
|
13
|
+
- fixed issue cutting tags/releasing
|
14
|
+
|
15
|
+
## 1.0.1
|
16
|
+
### Fixed
|
17
|
+
- fixed release to rubygems
|
18
|
+
|
19
|
+
## 1.0.0
|
20
|
+
### Changed
|
21
|
+
- Now hosted on rubygems
|
22
|
+
|
23
|
+
## 0.4.4 - 2019-3-4
|
24
|
+
### Fixed
|
25
|
+
- encrypted data of any type should receive the null obfuscator, previously only encrypted sensitive data was receiving null_obfuscator
|
26
|
+
|
27
|
+
## 0.4.3 - 2019-2-28
|
28
|
+
### Fixed
|
29
|
+
- catch ActiveRecord::NoDatabaseError for activation on new apps
|
30
|
+
|
31
|
+
## 0.4.2 - 2019-2-23
|
32
|
+
### Fixed
|
33
|
+
- removed require 'pry' line that was breaking in prod.
|
34
|
+
|
35
|
+
## 0.4.1 - 2019-2-15
|
36
|
+
### Fixed
|
37
|
+
- lack of space after curly brace in generated migration created lint errors
|
38
|
+
|
39
|
+
## 0.4.0 - 2019-2-15
|
40
|
+
### Added
|
41
|
+
- Added null_obfuscator for encrypted columns
|
42
|
+
- Added encryption check for sensitive data type
|
43
|
+
|
44
|
+
## 0.3.0 - 2019-2-15
|
45
|
+
### Added
|
46
|
+
- Added annotations for address columns
|
47
|
+
|
48
|
+
## 0.2.0 - 2019-2-15
|
49
|
+
### Added
|
50
|
+
- Added sensitive data columns like SIN, SSN, TIN
|
51
|
+
|
52
|
+
## 0.1.1 - 2019-2-14
|
53
|
+
### Fixed
|
54
|
+
- CircleCi database issue fixed
|
55
|
+
|
56
|
+
## 0.1.0 - 2019-2-14
|
57
|
+
### Added
|
58
|
+
- Colorize output in dev env and produce more descriptive message.
|
59
|
+
|
60
|
+
## 0.0.1 - 2019-2-14
|
61
|
+
### Added
|
62
|
+
- Gem created
|
data/Gemfile
ADDED
data/Guardfile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# A sample Guardfile
|
2
|
+
# More info at https://github.com/guard/guard#readme
|
3
|
+
|
4
|
+
## Uncomment and set this to only include directories you want to watch
|
5
|
+
# directories %w(app lib config test spec features) \
|
6
|
+
# .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
|
7
|
+
|
8
|
+
## Note: if you are using the `directories` clause above and you are not
|
9
|
+
## watching the project directory ('.'), then you will want to move
|
10
|
+
## the Guardfile to a watched dir and symlink it back, e.g.
|
11
|
+
#
|
12
|
+
# $ mkdir config
|
13
|
+
# $ mv Guardfile config/
|
14
|
+
# $ ln -s config/Guardfile .
|
15
|
+
#
|
16
|
+
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
|
17
|
+
|
18
|
+
guard :rspec, cmd: 'bundle exec rspec' do
|
19
|
+
require 'guard/rspec/dsl'
|
20
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
21
|
+
|
22
|
+
# Feel free to open issues for suggestions and improvements
|
23
|
+
|
24
|
+
# RSpec files
|
25
|
+
rspec = dsl.rspec
|
26
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
27
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
28
|
+
watch(rspec.spec_files)
|
29
|
+
|
30
|
+
# Ruby files
|
31
|
+
ruby = dsl.ruby
|
32
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
33
|
+
|
34
|
+
watch(/^(.+)\.rb$/) { |m| "spec/#{m[1]}_spec.rb" }
|
35
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
## PiiSafeSchema
|
2
|
+
|
3
|
+
this gem serves a few functions:
|
4
|
+
|
5
|
+
* Warning you when you might be missing an annotation on a column
|
6
|
+
* auto generating your migrations for you
|
7
|
+
* alerting the security team through datadog events if there are remaining unannotated columns
|
8
|
+
|
9
|
+
|
10
|
+
|
11
|
+
### Getting Started
|
12
|
+
|
13
|
+
`gem 'pii-safe-schema'`
|
14
|
+
|
15
|
+
add the following to `application.rb`
|
16
|
+
|
17
|
+
```
|
18
|
+
config.after_initialize do
|
19
|
+
PiiSafeSchema.activate!
|
20
|
+
end
|
21
|
+
```
|
22
|
+
|
23
|
+
if you want to ignore certain columns, add the following initializer:
|
24
|
+
|
25
|
+
```
|
26
|
+
# initializers/pii-safe-schema.rb
|
27
|
+
|
28
|
+
PiiSafeSchema.configure do |config|
|
29
|
+
config.ignore = {
|
30
|
+
some_table: :*, # ignore the whole table
|
31
|
+
some_other_table: [:column_1, :column_2] # just those columns
|
32
|
+
}
|
33
|
+
end
|
34
|
+
```
|
35
|
+
|
36
|
+
### Generating Comment Migrations
|
37
|
+
|
38
|
+
`rake pii_safe_schema:generate_migrations`
|
39
|
+
|
40
|
+
this will generate one migration file for each table that should be commented.
|
41
|
+
it will create a comment field for each column that it warns you about when you start a rails server or console.
|
42
|
+
|
43
|
+
|
44
|
+
|
45
|
+
|
46
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Annotations
|
3
|
+
SENSITIVE_DATA_NAMES = %w[
|
4
|
+
sin social_insurance_number ssn social_security_number
|
5
|
+
tin tax_idenfification_number national_insurance_number mifid
|
6
|
+
].freeze
|
7
|
+
COLUMNS = {
|
8
|
+
email: {
|
9
|
+
comment: {
|
10
|
+
pii: { obfuscate: 'email_obfuscator' },
|
11
|
+
},
|
12
|
+
regexp: /email/,
|
13
|
+
},
|
14
|
+
phone: {
|
15
|
+
comment: {
|
16
|
+
pii: { obfuscate: 'phone_obfuscator' },
|
17
|
+
},
|
18
|
+
regexp: /phone/,
|
19
|
+
},
|
20
|
+
ip_address: {
|
21
|
+
comment: {
|
22
|
+
pii: { obfuscate: 'ip_obfuscator' },
|
23
|
+
},
|
24
|
+
regexp: /ip_address/,
|
25
|
+
},
|
26
|
+
geolocation: {
|
27
|
+
comment: {
|
28
|
+
pii: { obfuscate: 'geo_obfuscator' },
|
29
|
+
},
|
30
|
+
regexp: /latitude|longitude/,
|
31
|
+
},
|
32
|
+
address: {
|
33
|
+
comment: {
|
34
|
+
pii: { obfuscate: 'null_obfuscator' },
|
35
|
+
},
|
36
|
+
regexp: /(^street|apt|apartment|unit_n)/,
|
37
|
+
},
|
38
|
+
postal_code: {
|
39
|
+
comment: {
|
40
|
+
pii: { obfuscate: 'postal_code_obfuscator' },
|
41
|
+
},
|
42
|
+
regexp: /(postal|zip)_code/,
|
43
|
+
},
|
44
|
+
name: {
|
45
|
+
comment: {
|
46
|
+
pii: { obfuscate: 'name_obfuscator' },
|
47
|
+
},
|
48
|
+
regexp: /(last|sur|full|^)_?(name)/,
|
49
|
+
},
|
50
|
+
sensitive_data: {
|
51
|
+
comment: {
|
52
|
+
pii: { tokenize: 'sha256_tokenizer' },
|
53
|
+
},
|
54
|
+
regexp: /(^|_)(#{SENSITIVE_DATA_NAMES.join("|")})(_|$)/,
|
55
|
+
},
|
56
|
+
encrypted_data: {
|
57
|
+
comment: {
|
58
|
+
pii: { obfuscate: 'null_obfuscator' },
|
59
|
+
},
|
60
|
+
regexp: /encrypted/,
|
61
|
+
},
|
62
|
+
}.freeze
|
63
|
+
|
64
|
+
def recommended_comment(column)
|
65
|
+
return COLUMNS[:encrypted_data][:comment] if apply_encrypted_recommendation?(column)
|
66
|
+
|
67
|
+
COLUMNS.each do |_type, info|
|
68
|
+
return info[:comment] if apply_recommendation?(column, info)
|
69
|
+
end
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
|
73
|
+
def apply_recommendation?(column, pii_info)
|
74
|
+
!encrypted?(column) &&
|
75
|
+
pii_info[:regexp].match(column.name) &&
|
76
|
+
column.comment != pii_info[:comment].to_json
|
77
|
+
end
|
78
|
+
|
79
|
+
def encrypted?(column)
|
80
|
+
COLUMNS[:encrypted_data][:regexp].match(column.name)
|
81
|
+
end
|
82
|
+
|
83
|
+
def apply_encrypted_recommendation?(column)
|
84
|
+
encrypted?(column) && column.comment != COLUMNS[:encrypted_data][:comment].to_json
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
class Configuration
|
3
|
+
DEFAULT_IGNORE = {
|
4
|
+
schema_migrations: :*,
|
5
|
+
ar_internal_metadata: :*,
|
6
|
+
}.freeze
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@user_ignore = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def ignore=(ignore_params)
|
13
|
+
validate(ignore_params)
|
14
|
+
@user_ignore = ignore_params
|
15
|
+
end
|
16
|
+
|
17
|
+
def ignore
|
18
|
+
@user_ignore.merge(DEFAULT_IGNORE)
|
19
|
+
end
|
20
|
+
|
21
|
+
def ignore_tables
|
22
|
+
ignore.select { |_k, v| v.to_s == '*' }.keys.map(&:to_s)
|
23
|
+
end
|
24
|
+
|
25
|
+
def ignore_columns
|
26
|
+
ignore.select { |_k, v| v.is_a?(Array) }
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def validate(ignore_params)
|
32
|
+
raise_config_error unless ignore_params.is_a?(Hash)
|
33
|
+
|
34
|
+
ignore_params.values.each do |ip|
|
35
|
+
raise_config_error unless valid_column_list?(ip) || ip == :*
|
36
|
+
end
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def valid_column_list?(value)
|
41
|
+
value.is_a?(Array) && value.all? { |c| c.is_a?(Symbol) }
|
42
|
+
end
|
43
|
+
|
44
|
+
def raise_config_error
|
45
|
+
raise ConfigurationError, ConfigurationError.message
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class ConfigurationError < StandardError
|
50
|
+
def self.message
|
51
|
+
<<~HEREDOC
|
52
|
+
ignore must be a hash where the values are
|
53
|
+
symbols or arrays of symbols.
|
54
|
+
e.g. ignore = { some_table: :* } ##ignore the whole some_table
|
55
|
+
or ignore = { some_table: [:some_column, :some_other_column] }
|
56
|
+
HEREDOC
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module MigrationGenerator
|
3
|
+
class << self
|
4
|
+
def generate_migrations(pii_columns)
|
5
|
+
pii_columns.group_by(&:table).map do |table, columns|
|
6
|
+
generate_migration_for(table, columns)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
# rubocop:disable Metrics/AbcSize
|
13
|
+
def generate_migration_for(table, columns)
|
14
|
+
generator = ActiveRecord::Generators::MigrationGenerator.new(
|
15
|
+
["change_comments_in_#{table}"],
|
16
|
+
)
|
17
|
+
generated_lines = generate_migration_lines(table, columns)
|
18
|
+
migration_file = generator.create_migration_file
|
19
|
+
file_lines = File.open(migration_file, 'r').read.split("\n")
|
20
|
+
change_line = file_lines.find_index { |i| /def change/.match(i) }
|
21
|
+
new_contents = file_lines[0..change_line] +
|
22
|
+
generated_lines +
|
23
|
+
file_lines[change_line + 1..-1]
|
24
|
+
|
25
|
+
File.open(migration_file, 'w') do |f|
|
26
|
+
f.write(new_contents.join("\n"))
|
27
|
+
f.write("\n")
|
28
|
+
end
|
29
|
+
migration_file
|
30
|
+
end
|
31
|
+
# rubocop:enable Metrics/AbcSize
|
32
|
+
|
33
|
+
def generate_migration_lines(table, columns)
|
34
|
+
migration_lines = columns.map do |c|
|
35
|
+
"#{' ' * (safety_assured? ? 6 : 4)}"\
|
36
|
+
"change_column :#{table}, :#{c.column.name}, :#{c.column.type}, "\
|
37
|
+
"comment: \'#{c.suggestion.to_json}\'"\
|
38
|
+
end
|
39
|
+
wrap_in_safety_assured(migration_lines)
|
40
|
+
end
|
41
|
+
|
42
|
+
def wrap_in_safety_assured(migration_lines)
|
43
|
+
return migration_lines unless safety_assured?
|
44
|
+
|
45
|
+
["#{' ' * 4}safety_assured do", *migration_lines, "#{' ' * 4}end"]
|
46
|
+
end
|
47
|
+
|
48
|
+
def safety_assured?
|
49
|
+
defined?(StrongMigrations)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Notify
|
3
|
+
module DataDog
|
4
|
+
KNOWN_CLIENTS = %w[DataDogClient Ws::Railway::Datadog].freeze
|
5
|
+
|
6
|
+
class << self
|
7
|
+
def deliver(pii_column)
|
8
|
+
return unless %w[staging production development].include?(Rails.env)
|
9
|
+
return if dog_client.nil?
|
10
|
+
|
11
|
+
dog_client.event('PII Annotation Warning',
|
12
|
+
message(pii_column),
|
13
|
+
msg_title: 'Unannotated PII Column',
|
14
|
+
alert_type: 'warning')
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def message(pii_column)
|
20
|
+
"column #{pii_column.table}.#{pii_column.column.name} is not annotated"
|
21
|
+
end
|
22
|
+
|
23
|
+
def dog_client
|
24
|
+
KNOWN_CLIENTS.each do |client|
|
25
|
+
return client.safe_constantize if defined?(client)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
module PiiSafeSchema
|
3
|
+
module Notify
|
4
|
+
module StdOut
|
5
|
+
class << self
|
6
|
+
def deliver(pii_column)
|
7
|
+
Rails.logger.info(message(pii_column).red)
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def message(pii_column)
|
13
|
+
<<~HEREDOC
|
14
|
+
------------------------------------------------------------------------------------
|
15
|
+
Annotation recommended on column:
|
16
|
+
#{pii_column.table}.#{pii_column.column.name}: comment: \"#{pii_column.suggestion}\"
|
17
|
+
|
18
|
+
run `rake pii_safe_schema:generate_migrations`
|
19
|
+
to generate all necessary annotation migrations.
|
20
|
+
|
21
|
+
if this column does not contain PII, you can ignore it
|
22
|
+
in your PiiSafeSchema configs.
|
23
|
+
https://github.com/wealthsimple/pii-safe-schema/blob/master/README.md
|
24
|
+
------------------------------------------------------------------------------------
|
25
|
+
HEREDOC
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Notify
|
3
|
+
METHODS = %i[StdOut DataDog].freeze
|
4
|
+
def self.notify(column_or_columns)
|
5
|
+
column_or_columns.each { |c| deliver(c) } if column_or_columns.is_a?(Array)
|
6
|
+
deliver(c) if column_or_columns.is_a?(PiiSafeSchema::PiiColumn)
|
7
|
+
end
|
8
|
+
|
9
|
+
class << self
|
10
|
+
private
|
11
|
+
|
12
|
+
def deliver(pii_column)
|
13
|
+
METHODS.each do |m|
|
14
|
+
"PiiSafeSchema::Notify::#{m}".constantize.deliver(pii_column)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
class PiiColumn
|
3
|
+
extend PiiSafeSchema::Annotations
|
4
|
+
attr_reader :table, :column, :suggestion
|
5
|
+
|
6
|
+
def self.all
|
7
|
+
find_and_create
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize(table:, column:, suggestion:)
|
11
|
+
@table = table.to_sym
|
12
|
+
@column = column
|
13
|
+
@suggestion = suggestion
|
14
|
+
end
|
15
|
+
|
16
|
+
class << self
|
17
|
+
private
|
18
|
+
|
19
|
+
def find_and_create
|
20
|
+
relevant_tables.map do |table|
|
21
|
+
connection.columns(table).map do |column|
|
22
|
+
next if ignored_column?(table, column)
|
23
|
+
|
24
|
+
rec = recommended_comment(column)
|
25
|
+
rec ? new(table: table, column: column, suggestion: rec) : nil
|
26
|
+
end.compact
|
27
|
+
end.compact.flatten
|
28
|
+
end
|
29
|
+
|
30
|
+
def connection
|
31
|
+
ActiveRecord::Base.connection
|
32
|
+
end
|
33
|
+
|
34
|
+
def relevant_tables
|
35
|
+
connection.tables - PiiSafeSchema.configuration.ignore_tables
|
36
|
+
end
|
37
|
+
|
38
|
+
def ignored_column?(table, column)
|
39
|
+
PiiSafeSchema.configuration.
|
40
|
+
ignore_columns[table.to_sym]&.
|
41
|
+
include?(column.name.to_sym)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'pii_safe_schema/configuration'
|
2
|
+
require 'pii_safe_schema/annotations'
|
3
|
+
require 'pii_safe_schema/notify'
|
4
|
+
require 'pii_safe_schema/pii_column'
|
5
|
+
require 'pii_safe_schema/version'
|
6
|
+
require 'pii_safe_schema/notifiers/std_out'
|
7
|
+
require 'pii_safe_schema/notifiers/data_dog'
|
8
|
+
require 'pii_safe_schema/railtie'
|
9
|
+
require 'rails/generators'
|
10
|
+
require 'rails/generators/active_record/migration/migration_generator'
|
11
|
+
require 'pii_safe_schema/migration_generator'
|
12
|
+
require 'json'
|
13
|
+
|
14
|
+
module PiiSafeSchema
|
15
|
+
extend PiiSafeSchema::Notify
|
16
|
+
|
17
|
+
def self.configuration
|
18
|
+
@configuration ||= Configuration.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.configure
|
22
|
+
yield(configuration)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.activate!
|
26
|
+
return if Rails.env.test?
|
27
|
+
|
28
|
+
ActiveSupport.on_load :active_record do
|
29
|
+
Notify.notify(PiiSafeSchema::PiiColumn.all)
|
30
|
+
end
|
31
|
+
rescue ActiveRecord::NoDatabaseError
|
32
|
+
Rails.logger.info('PiiSafeSchema: No DB'.red)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.generate_migrations
|
36
|
+
PiiSafeSchema::MigrationGenerator.generate_migrations(PiiSafeSchema::PiiColumn.all)
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'pii_safe_schema/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'pii_safe_schema'
|
7
|
+
s.version = PiiSafeSchema::VERSION
|
8
|
+
s.authors = ['Alexi Garrow']
|
9
|
+
s.email = ['agarrow@wealthsimple.com']
|
10
|
+
|
11
|
+
s.summary = 'Schema migration tool for checking and adding comments on PII columns.'
|
12
|
+
s.homepage = 'https://github.com/wealthsimple/pii-safe-schema'
|
13
|
+
|
14
|
+
s.files = `git ls-files -z`.split("\x0").reject do |f|
|
15
|
+
f.match(%r{^(test|spec|features)/})
|
16
|
+
end
|
17
|
+
|
18
|
+
s.bindir = 'exe'
|
19
|
+
s.executables = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
s.require_paths = ['lib']
|
21
|
+
|
22
|
+
s.add_dependency 'activesupport', '>= 5'
|
23
|
+
s.add_dependency 'colorize'
|
24
|
+
s.add_dependency 'rails', '>= 5'
|
25
|
+
|
26
|
+
s.add_development_dependency 'bundler', '~> 1.16'
|
27
|
+
s.add_development_dependency 'bundler-audit'
|
28
|
+
s.add_development_dependency 'dogstatsd-ruby'
|
29
|
+
s.add_development_dependency 'git'
|
30
|
+
s.add_development_dependency 'guard-rspec'
|
31
|
+
s.add_development_dependency 'pry'
|
32
|
+
s.add_development_dependency 'rake', '~> 10.0'
|
33
|
+
s.add_development_dependency 'rspec', '~> 3.0'
|
34
|
+
s.add_development_dependency 'rspec-collection_matchers'
|
35
|
+
s.add_development_dependency 'rspec-its'
|
36
|
+
s.add_development_dependency 'rubocop'
|
37
|
+
s.add_development_dependency 'simplecov'
|
38
|
+
s.add_development_dependency 'ws-style'
|
39
|
+
|
40
|
+
# Required by activerecord-safer_migrations
|
41
|
+
s.add_development_dependency 'pg', '~> 0.21'
|
42
|
+
s.add_development_dependency 'strong_migrations'
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,318 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pii_safe_schema
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alexi Garrow
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-04-16 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: colorize
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rails
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '5'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '5'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.16'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.16'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler-audit
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: dogstatsd-ruby
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: git
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: guard-rspec
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: rake
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '10.0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '10.0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rspec
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '3.0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '3.0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rspec-collection_matchers
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec-its
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: rubocop
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
209
|
+
- !ruby/object:Gem::Dependency
|
210
|
+
name: simplecov
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
212
|
+
requirements:
|
213
|
+
- - ">="
|
214
|
+
- !ruby/object:Gem::Version
|
215
|
+
version: '0'
|
216
|
+
type: :development
|
217
|
+
prerelease: false
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - ">="
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: '0'
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: ws-style
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - ">="
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: '0'
|
237
|
+
- !ruby/object:Gem::Dependency
|
238
|
+
name: pg
|
239
|
+
requirement: !ruby/object:Gem::Requirement
|
240
|
+
requirements:
|
241
|
+
- - "~>"
|
242
|
+
- !ruby/object:Gem::Version
|
243
|
+
version: '0.21'
|
244
|
+
type: :development
|
245
|
+
prerelease: false
|
246
|
+
version_requirements: !ruby/object:Gem::Requirement
|
247
|
+
requirements:
|
248
|
+
- - "~>"
|
249
|
+
- !ruby/object:Gem::Version
|
250
|
+
version: '0.21'
|
251
|
+
- !ruby/object:Gem::Dependency
|
252
|
+
name: strong_migrations
|
253
|
+
requirement: !ruby/object:Gem::Requirement
|
254
|
+
requirements:
|
255
|
+
- - ">="
|
256
|
+
- !ruby/object:Gem::Version
|
257
|
+
version: '0'
|
258
|
+
type: :development
|
259
|
+
prerelease: false
|
260
|
+
version_requirements: !ruby/object:Gem::Requirement
|
261
|
+
requirements:
|
262
|
+
- - ">="
|
263
|
+
- !ruby/object:Gem::Version
|
264
|
+
version: '0'
|
265
|
+
description:
|
266
|
+
email:
|
267
|
+
- agarrow@wealthsimple.com
|
268
|
+
executables: []
|
269
|
+
extensions: []
|
270
|
+
extra_rdoc_files: []
|
271
|
+
files:
|
272
|
+
- ".circleci/config.yml"
|
273
|
+
- ".github/CODEOWNERS"
|
274
|
+
- ".github/PULL_REQUEST_TEMPLATE.md"
|
275
|
+
- ".gitignore"
|
276
|
+
- ".rspec"
|
277
|
+
- ".rubocop.yml"
|
278
|
+
- ".ruby-version"
|
279
|
+
- CHANGELOG.md
|
280
|
+
- Gemfile
|
281
|
+
- Guardfile
|
282
|
+
- README.md
|
283
|
+
- Rakefile
|
284
|
+
- lib/pii_safe_schema.rb
|
285
|
+
- lib/pii_safe_schema/annotations.rb
|
286
|
+
- lib/pii_safe_schema/configuration.rb
|
287
|
+
- lib/pii_safe_schema/migration_generator.rb
|
288
|
+
- lib/pii_safe_schema/notifiers/data_dog.rb
|
289
|
+
- lib/pii_safe_schema/notifiers/std_out.rb
|
290
|
+
- lib/pii_safe_schema/notify.rb
|
291
|
+
- lib/pii_safe_schema/pii_column.rb
|
292
|
+
- lib/pii_safe_schema/railtie.rb
|
293
|
+
- lib/pii_safe_schema/version.rb
|
294
|
+
- lib/tasks/pii_safe_schema.rake
|
295
|
+
- pii-safe-schema.gemspec
|
296
|
+
homepage: https://github.com/wealthsimple/pii-safe-schema
|
297
|
+
licenses: []
|
298
|
+
metadata: {}
|
299
|
+
post_install_message:
|
300
|
+
rdoc_options: []
|
301
|
+
require_paths:
|
302
|
+
- lib
|
303
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
304
|
+
requirements:
|
305
|
+
- - ">="
|
306
|
+
- !ruby/object:Gem::Version
|
307
|
+
version: '0'
|
308
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
309
|
+
requirements:
|
310
|
+
- - ">="
|
311
|
+
- !ruby/object:Gem::Version
|
312
|
+
version: '0'
|
313
|
+
requirements: []
|
314
|
+
rubygems_version: 3.0.1
|
315
|
+
signing_key:
|
316
|
+
specification_version: 4
|
317
|
+
summary: Schema migration tool for checking and adding comments on PII columns.
|
318
|
+
test_files: []
|