pii_safe_schema 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +118 -0
- data/.github/CODEOWNERS +1 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +16 -0
- data/.gitignore +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +3 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +62 -0
- data/Gemfile +3 -0
- data/Guardfile +35 -0
- data/README.md +46 -0
- data/Rakefile +8 -0
- data/lib/pii_safe_schema/annotations.rb +87 -0
- data/lib/pii_safe_schema/configuration.rb +59 -0
- data/lib/pii_safe_schema/migration_generator.rb +53 -0
- data/lib/pii_safe_schema/notifiers/data_dog.rb +31 -0
- data/lib/pii_safe_schema/notifiers/std_out.rb +30 -0
- data/lib/pii_safe_schema/notify.rb +19 -0
- data/lib/pii_safe_schema/pii_column.rb +45 -0
- data/lib/pii_safe_schema/railtie.rb +9 -0
- data/lib/pii_safe_schema/version.rb +3 -0
- data/lib/pii_safe_schema.rb +38 -0
- data/lib/tasks/pii_safe_schema.rake +5 -0
- data/pii-safe-schema.gemspec +43 -0
- metadata +318 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 8bca88cff66431d4ed83b6f8e3956e40686d858b47beac08a0efe05eae5a8b42
|
4
|
+
data.tar.gz: 8ab4b451afbf182da050c29e6e4f8d9ea00cdd225940604b242530db6f8d3649
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5e2ddf4cbe30dbe6285581508e4d138de72acf42f031045879cf85f74a28501970f4bb7a3af3dc612e1d4eef248d94154c8c8149e6d3af79ac63b2766fdb24e6
|
7
|
+
data.tar.gz: 625c1edd58177ce1f79fada40a3e25bd1701133516ef73579b40496e81e4cdbaa97368f74cbec1d7098b0e8c23969d264cfc2b16ac140dbaa61a03b840b712cc
|
@@ -0,0 +1,118 @@
|
|
1
|
+
version: 2
|
2
|
+
|
3
|
+
defaults: &defaults
|
4
|
+
working_directory: /home/circleci/wealthsimple
|
5
|
+
docker:
|
6
|
+
- image: circleci/ruby:2.6.0
|
7
|
+
- image: circleci/postgres:9.5.9-alpine
|
8
|
+
environment:
|
9
|
+
POSTGRES_USER: circleci
|
10
|
+
POSTGRES_DB: pii_safe_schema_test
|
11
|
+
|
12
|
+
# These are common snippets that are referenced in multiple workflows.
|
13
|
+
references:
|
14
|
+
attach_code_workspace: &attach_code_workspace
|
15
|
+
attach_workspace:
|
16
|
+
at: /home/circleci/wealthsimple
|
17
|
+
|
18
|
+
restore_bundle_dependencies: &restore_bundle_dependencies
|
19
|
+
run:
|
20
|
+
name: Restore bundle dependencies from workspace
|
21
|
+
command: bundle --path vendor/bundle
|
22
|
+
|
23
|
+
jobs:
|
24
|
+
checkout_and_bundle:
|
25
|
+
<<: *defaults
|
26
|
+
steps:
|
27
|
+
- checkout
|
28
|
+
- run:
|
29
|
+
command: bundle install --jobs=4 --retry=3 --path vendor/bundle
|
30
|
+
- persist_to_workspace:
|
31
|
+
root: .
|
32
|
+
paths: .
|
33
|
+
|
34
|
+
rspec:
|
35
|
+
<<: *defaults
|
36
|
+
steps:
|
37
|
+
- *attach_code_workspace
|
38
|
+
- *restore_bundle_dependencies
|
39
|
+
- run:
|
40
|
+
command: sudo apt install -y postgresql-client || true
|
41
|
+
- run:
|
42
|
+
command: bundle exec bundle-audit update && bundle exec bundle-audit check
|
43
|
+
- run:
|
44
|
+
command: bundle exec rspec
|
45
|
+
|
46
|
+
lint_check:
|
47
|
+
<<: *defaults
|
48
|
+
steps:
|
49
|
+
- *attach_code_workspace
|
50
|
+
- *restore_bundle_dependencies
|
51
|
+
- run:
|
52
|
+
command: bundle exec rubocop
|
53
|
+
|
54
|
+
vulnerability_check:
|
55
|
+
<<: *defaults
|
56
|
+
steps:
|
57
|
+
- *attach_code_workspace
|
58
|
+
- *restore_bundle_dependencies
|
59
|
+
- run:
|
60
|
+
command: bundle exec bundle-audit update && bundle exec bundle-audit check
|
61
|
+
|
62
|
+
release:
|
63
|
+
<<: *defaults
|
64
|
+
steps:
|
65
|
+
- add_ssh_keys:
|
66
|
+
fingerprints:
|
67
|
+
- "46:b5:cb:ee:57:dc:14:95:31:be:12:13:4f:11:94:a4"
|
68
|
+
- *attach_code_workspace
|
69
|
+
- *restore_bundle_dependencies
|
70
|
+
- run:
|
71
|
+
name: Release to rubygems.org
|
72
|
+
command: |
|
73
|
+
mkdir ~/.gem
|
74
|
+
echo ":rubygems_api_key: ${RUBYGEMS_API_KEY}" >> ~/.gem/credentials
|
75
|
+
chmod 600 ~/.gem/credentials
|
76
|
+
mkdir -p ~/.ssh
|
77
|
+
echo "github.com,192.30.253.112 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> ~/.ssh/known_hosts
|
78
|
+
bundle exec rake release
|
79
|
+
|
80
|
+
workflows:
|
81
|
+
version: 2
|
82
|
+
build_and_test:
|
83
|
+
jobs:
|
84
|
+
- checkout_and_bundle:
|
85
|
+
context: wealthsimple
|
86
|
+
- rspec:
|
87
|
+
requires:
|
88
|
+
- checkout_and_bundle
|
89
|
+
- lint_check:
|
90
|
+
requires:
|
91
|
+
- checkout_and_bundle
|
92
|
+
- vulnerability_check:
|
93
|
+
requires:
|
94
|
+
- checkout_and_bundle
|
95
|
+
- release:
|
96
|
+
context: wealthsimple
|
97
|
+
filters:
|
98
|
+
branches:
|
99
|
+
only: master
|
100
|
+
requires:
|
101
|
+
- rspec
|
102
|
+
- lint_check
|
103
|
+
- vulnerability_check
|
104
|
+
|
105
|
+
security-audit:
|
106
|
+
triggers:
|
107
|
+
- schedule:
|
108
|
+
# 11:45 am UTC: 6:45 am EST / 7:45 am EDT
|
109
|
+
cron: "45 11 * * *"
|
110
|
+
filters:
|
111
|
+
branches:
|
112
|
+
only: master
|
113
|
+
jobs:
|
114
|
+
- checkout_and_bundle:
|
115
|
+
context: wealthsimple
|
116
|
+
- vulnerability_check:
|
117
|
+
requires:
|
118
|
+
- checkout_and_bundle
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @wealthsimple/security
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#### Why <!-- A short description of why this change is required -->
|
2
|
+
|
3
|
+
|
4
|
+
|
5
|
+
#### What changed <!-- Summary of changes when modifying hundreds of lines -->
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
<!--
|
10
|
+
Consider adding the following sections:
|
11
|
+
|
12
|
+
#### How I tested [ Bullets for test cases covered ]
|
13
|
+
#### Next steps [ If your PR is part of a few or a WIP, give context to reviewers ]
|
14
|
+
#### Screenshot [ An image is worth a thousand words ]
|
15
|
+
#### Bug/Ticket tracker [ Unnecessary when prefixing branch with JIRA ticket, e.g. SECURITY-123-human-readable-thing ]
|
16
|
+
-->
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.6.0
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
# Changelog
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
|
+
|
7
|
+
## 1.0.3
|
8
|
+
### Fixed
|
9
|
+
- bumping version to release to rubygems
|
10
|
+
|
11
|
+
## 1.0.2
|
12
|
+
### Fixed
|
13
|
+
- fixed issue cutting tags/releasing
|
14
|
+
|
15
|
+
## 1.0.1
|
16
|
+
### Fixed
|
17
|
+
- fixed release to rubygems
|
18
|
+
|
19
|
+
## 1.0.0
|
20
|
+
### Changed
|
21
|
+
- Now hosted on rubygems
|
22
|
+
|
23
|
+
## 0.4.4 - 2019-3-4
|
24
|
+
### Fixed
|
25
|
+
- encrypted data of any type should receive the null obfuscator, previously only encrypted sensitive data was receiving null_obfuscator
|
26
|
+
|
27
|
+
## 0.4.3 - 2019-2-28
|
28
|
+
### Fixed
|
29
|
+
- catch ActiveRecord::NoDatabaseError for activation on new apps
|
30
|
+
|
31
|
+
## 0.4.2 - 2019-2-23
|
32
|
+
### Fixed
|
33
|
+
- removed require 'pry' line that was breaking in prod.
|
34
|
+
|
35
|
+
## 0.4.1 - 2019-2-15
|
36
|
+
### Fixed
|
37
|
+
- lack of space after curly brace in generated migration created lint errors
|
38
|
+
|
39
|
+
## 0.4.0 - 2019-2-15
|
40
|
+
### Added
|
41
|
+
- Added null_obfuscator for encrypted columns
|
42
|
+
- Added encryption check for sensitive data type
|
43
|
+
|
44
|
+
## 0.3.0 - 2019-2-15
|
45
|
+
### Added
|
46
|
+
- Added annotations for address columns
|
47
|
+
|
48
|
+
## 0.2.0 - 2019-2-15
|
49
|
+
### Added
|
50
|
+
- Added sensitive data columns like SIN, SSN, TIN
|
51
|
+
|
52
|
+
## 0.1.1 - 2019-2-14
|
53
|
+
### Fixed
|
54
|
+
- CircleCi database issue fixed
|
55
|
+
|
56
|
+
## 0.1.0 - 2019-2-14
|
57
|
+
### Added
|
58
|
+
- Colorize output in dev env and produce more descriptive message.
|
59
|
+
|
60
|
+
## 0.0.1 - 2019-2-14
|
61
|
+
### Added
|
62
|
+
- Gem created
|
data/Gemfile
ADDED
data/Guardfile
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# A sample Guardfile
|
2
|
+
# More info at https://github.com/guard/guard#readme
|
3
|
+
|
4
|
+
## Uncomment and set this to only include directories you want to watch
|
5
|
+
# directories %w(app lib config test spec features) \
|
6
|
+
# .select{|d| Dir.exist?(d) ? d : UI.warning("Directory #{d} does not exist")}
|
7
|
+
|
8
|
+
## Note: if you are using the `directories` clause above and you are not
|
9
|
+
## watching the project directory ('.'), then you will want to move
|
10
|
+
## the Guardfile to a watched dir and symlink it back, e.g.
|
11
|
+
#
|
12
|
+
# $ mkdir config
|
13
|
+
# $ mv Guardfile config/
|
14
|
+
# $ ln -s config/Guardfile .
|
15
|
+
#
|
16
|
+
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
|
17
|
+
|
18
|
+
guard :rspec, cmd: 'bundle exec rspec' do
|
19
|
+
require 'guard/rspec/dsl'
|
20
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
21
|
+
|
22
|
+
# Feel free to open issues for suggestions and improvements
|
23
|
+
|
24
|
+
# RSpec files
|
25
|
+
rspec = dsl.rspec
|
26
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
27
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
28
|
+
watch(rspec.spec_files)
|
29
|
+
|
30
|
+
# Ruby files
|
31
|
+
ruby = dsl.ruby
|
32
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
33
|
+
|
34
|
+
watch(/^(.+)\.rb$/) { |m| "spec/#{m[1]}_spec.rb" }
|
35
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
## PiiSafeSchema
|
2
|
+
|
3
|
+
this gem serves a few functions:
|
4
|
+
|
5
|
+
* Warning you when you might be missing an annotation on a column
|
6
|
+
* auto generating your migrations for you
|
7
|
+
* alerting the security team through datadog events if there are remaining unannotated columns
|
8
|
+
|
9
|
+
|
10
|
+
|
11
|
+
### Getting Started
|
12
|
+
|
13
|
+
`gem 'pii-safe-schema'`
|
14
|
+
|
15
|
+
add the following to `application.rb`
|
16
|
+
|
17
|
+
```
|
18
|
+
config.after_initialize do
|
19
|
+
PiiSafeSchema.activate!
|
20
|
+
end
|
21
|
+
```
|
22
|
+
|
23
|
+
if you want to ignore certain columns, add the following initializer:
|
24
|
+
|
25
|
+
```
|
26
|
+
# initializers/pii-safe-schema.rb
|
27
|
+
|
28
|
+
PiiSafeSchema.configure do |config|
|
29
|
+
config.ignore = {
|
30
|
+
some_table: :*, # ignore the whole table
|
31
|
+
some_other_table: [:column_1, :column_2] # just those columns
|
32
|
+
}
|
33
|
+
end
|
34
|
+
```
|
35
|
+
|
36
|
+
### Generating Comment Migrations
|
37
|
+
|
38
|
+
`rake pii_safe_schema:generate_migrations`
|
39
|
+
|
40
|
+
this will generate one migration file for each table that should be commented.
|
41
|
+
it will create a comment field for each column that it warns you about when you start a rails server or console.
|
42
|
+
|
43
|
+
|
44
|
+
|
45
|
+
|
46
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Annotations
|
3
|
+
SENSITIVE_DATA_NAMES = %w[
|
4
|
+
sin social_insurance_number ssn social_security_number
|
5
|
+
tin tax_idenfification_number national_insurance_number mifid
|
6
|
+
].freeze
|
7
|
+
COLUMNS = {
|
8
|
+
email: {
|
9
|
+
comment: {
|
10
|
+
pii: { obfuscate: 'email_obfuscator' },
|
11
|
+
},
|
12
|
+
regexp: /email/,
|
13
|
+
},
|
14
|
+
phone: {
|
15
|
+
comment: {
|
16
|
+
pii: { obfuscate: 'phone_obfuscator' },
|
17
|
+
},
|
18
|
+
regexp: /phone/,
|
19
|
+
},
|
20
|
+
ip_address: {
|
21
|
+
comment: {
|
22
|
+
pii: { obfuscate: 'ip_obfuscator' },
|
23
|
+
},
|
24
|
+
regexp: /ip_address/,
|
25
|
+
},
|
26
|
+
geolocation: {
|
27
|
+
comment: {
|
28
|
+
pii: { obfuscate: 'geo_obfuscator' },
|
29
|
+
},
|
30
|
+
regexp: /latitude|longitude/,
|
31
|
+
},
|
32
|
+
address: {
|
33
|
+
comment: {
|
34
|
+
pii: { obfuscate: 'null_obfuscator' },
|
35
|
+
},
|
36
|
+
regexp: /(^street|apt|apartment|unit_n)/,
|
37
|
+
},
|
38
|
+
postal_code: {
|
39
|
+
comment: {
|
40
|
+
pii: { obfuscate: 'postal_code_obfuscator' },
|
41
|
+
},
|
42
|
+
regexp: /(postal|zip)_code/,
|
43
|
+
},
|
44
|
+
name: {
|
45
|
+
comment: {
|
46
|
+
pii: { obfuscate: 'name_obfuscator' },
|
47
|
+
},
|
48
|
+
regexp: /(last|sur|full|^)_?(name)/,
|
49
|
+
},
|
50
|
+
sensitive_data: {
|
51
|
+
comment: {
|
52
|
+
pii: { tokenize: 'sha256_tokenizer' },
|
53
|
+
},
|
54
|
+
regexp: /(^|_)(#{SENSITIVE_DATA_NAMES.join("|")})(_|$)/,
|
55
|
+
},
|
56
|
+
encrypted_data: {
|
57
|
+
comment: {
|
58
|
+
pii: { obfuscate: 'null_obfuscator' },
|
59
|
+
},
|
60
|
+
regexp: /encrypted/,
|
61
|
+
},
|
62
|
+
}.freeze
|
63
|
+
|
64
|
+
def recommended_comment(column)
|
65
|
+
return COLUMNS[:encrypted_data][:comment] if apply_encrypted_recommendation?(column)
|
66
|
+
|
67
|
+
COLUMNS.each do |_type, info|
|
68
|
+
return info[:comment] if apply_recommendation?(column, info)
|
69
|
+
end
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
|
73
|
+
def apply_recommendation?(column, pii_info)
|
74
|
+
!encrypted?(column) &&
|
75
|
+
pii_info[:regexp].match(column.name) &&
|
76
|
+
column.comment != pii_info[:comment].to_json
|
77
|
+
end
|
78
|
+
|
79
|
+
def encrypted?(column)
|
80
|
+
COLUMNS[:encrypted_data][:regexp].match(column.name)
|
81
|
+
end
|
82
|
+
|
83
|
+
def apply_encrypted_recommendation?(column)
|
84
|
+
encrypted?(column) && column.comment != COLUMNS[:encrypted_data][:comment].to_json
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
class Configuration
|
3
|
+
DEFAULT_IGNORE = {
|
4
|
+
schema_migrations: :*,
|
5
|
+
ar_internal_metadata: :*,
|
6
|
+
}.freeze
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@user_ignore = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def ignore=(ignore_params)
|
13
|
+
validate(ignore_params)
|
14
|
+
@user_ignore = ignore_params
|
15
|
+
end
|
16
|
+
|
17
|
+
def ignore
|
18
|
+
@user_ignore.merge(DEFAULT_IGNORE)
|
19
|
+
end
|
20
|
+
|
21
|
+
def ignore_tables
|
22
|
+
ignore.select { |_k, v| v.to_s == '*' }.keys.map(&:to_s)
|
23
|
+
end
|
24
|
+
|
25
|
+
def ignore_columns
|
26
|
+
ignore.select { |_k, v| v.is_a?(Array) }
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def validate(ignore_params)
|
32
|
+
raise_config_error unless ignore_params.is_a?(Hash)
|
33
|
+
|
34
|
+
ignore_params.values.each do |ip|
|
35
|
+
raise_config_error unless valid_column_list?(ip) || ip == :*
|
36
|
+
end
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def valid_column_list?(value)
|
41
|
+
value.is_a?(Array) && value.all? { |c| c.is_a?(Symbol) }
|
42
|
+
end
|
43
|
+
|
44
|
+
def raise_config_error
|
45
|
+
raise ConfigurationError, ConfigurationError.message
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class ConfigurationError < StandardError
|
50
|
+
def self.message
|
51
|
+
<<~HEREDOC
|
52
|
+
ignore must be a hash where the values are
|
53
|
+
symbols or arrays of symbols.
|
54
|
+
e.g. ignore = { some_table: :* } ##ignore the whole some_table
|
55
|
+
or ignore = { some_table: [:some_column, :some_other_column] }
|
56
|
+
HEREDOC
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module MigrationGenerator
|
3
|
+
class << self
|
4
|
+
def generate_migrations(pii_columns)
|
5
|
+
pii_columns.group_by(&:table).map do |table, columns|
|
6
|
+
generate_migration_for(table, columns)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
# rubocop:disable Metrics/AbcSize
|
13
|
+
def generate_migration_for(table, columns)
|
14
|
+
generator = ActiveRecord::Generators::MigrationGenerator.new(
|
15
|
+
["change_comments_in_#{table}"],
|
16
|
+
)
|
17
|
+
generated_lines = generate_migration_lines(table, columns)
|
18
|
+
migration_file = generator.create_migration_file
|
19
|
+
file_lines = File.open(migration_file, 'r').read.split("\n")
|
20
|
+
change_line = file_lines.find_index { |i| /def change/.match(i) }
|
21
|
+
new_contents = file_lines[0..change_line] +
|
22
|
+
generated_lines +
|
23
|
+
file_lines[change_line + 1..-1]
|
24
|
+
|
25
|
+
File.open(migration_file, 'w') do |f|
|
26
|
+
f.write(new_contents.join("\n"))
|
27
|
+
f.write("\n")
|
28
|
+
end
|
29
|
+
migration_file
|
30
|
+
end
|
31
|
+
# rubocop:enable Metrics/AbcSize
|
32
|
+
|
33
|
+
def generate_migration_lines(table, columns)
|
34
|
+
migration_lines = columns.map do |c|
|
35
|
+
"#{' ' * (safety_assured? ? 6 : 4)}"\
|
36
|
+
"change_column :#{table}, :#{c.column.name}, :#{c.column.type}, "\
|
37
|
+
"comment: \'#{c.suggestion.to_json}\'"\
|
38
|
+
end
|
39
|
+
wrap_in_safety_assured(migration_lines)
|
40
|
+
end
|
41
|
+
|
42
|
+
def wrap_in_safety_assured(migration_lines)
|
43
|
+
return migration_lines unless safety_assured?
|
44
|
+
|
45
|
+
["#{' ' * 4}safety_assured do", *migration_lines, "#{' ' * 4}end"]
|
46
|
+
end
|
47
|
+
|
48
|
+
def safety_assured?
|
49
|
+
defined?(StrongMigrations)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Notify
|
3
|
+
module DataDog
|
4
|
+
KNOWN_CLIENTS = %w[DataDogClient Ws::Railway::Datadog].freeze
|
5
|
+
|
6
|
+
class << self
|
7
|
+
def deliver(pii_column)
|
8
|
+
return unless %w[staging production development].include?(Rails.env)
|
9
|
+
return if dog_client.nil?
|
10
|
+
|
11
|
+
dog_client.event('PII Annotation Warning',
|
12
|
+
message(pii_column),
|
13
|
+
msg_title: 'Unannotated PII Column',
|
14
|
+
alert_type: 'warning')
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def message(pii_column)
|
20
|
+
"column #{pii_column.table}.#{pii_column.column.name} is not annotated"
|
21
|
+
end
|
22
|
+
|
23
|
+
def dog_client
|
24
|
+
KNOWN_CLIENTS.each do |client|
|
25
|
+
return client.safe_constantize if defined?(client)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
module PiiSafeSchema
|
3
|
+
module Notify
|
4
|
+
module StdOut
|
5
|
+
class << self
|
6
|
+
def deliver(pii_column)
|
7
|
+
Rails.logger.info(message(pii_column).red)
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def message(pii_column)
|
13
|
+
<<~HEREDOC
|
14
|
+
------------------------------------------------------------------------------------
|
15
|
+
Annotation recommended on column:
|
16
|
+
#{pii_column.table}.#{pii_column.column.name}: comment: \"#{pii_column.suggestion}\"
|
17
|
+
|
18
|
+
run `rake pii_safe_schema:generate_migrations`
|
19
|
+
to generate all necessary annotation migrations.
|
20
|
+
|
21
|
+
if this column does not contain PII, you can ignore it
|
22
|
+
in your PiiSafeSchema configs.
|
23
|
+
https://github.com/wealthsimple/pii-safe-schema/blob/master/README.md
|
24
|
+
------------------------------------------------------------------------------------
|
25
|
+
HEREDOC
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
module Notify
|
3
|
+
METHODS = %i[StdOut DataDog].freeze
|
4
|
+
def self.notify(column_or_columns)
|
5
|
+
column_or_columns.each { |c| deliver(c) } if column_or_columns.is_a?(Array)
|
6
|
+
deliver(c) if column_or_columns.is_a?(PiiSafeSchema::PiiColumn)
|
7
|
+
end
|
8
|
+
|
9
|
+
class << self
|
10
|
+
private
|
11
|
+
|
12
|
+
def deliver(pii_column)
|
13
|
+
METHODS.each do |m|
|
14
|
+
"PiiSafeSchema::Notify::#{m}".constantize.deliver(pii_column)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module PiiSafeSchema
|
2
|
+
class PiiColumn
|
3
|
+
extend PiiSafeSchema::Annotations
|
4
|
+
attr_reader :table, :column, :suggestion
|
5
|
+
|
6
|
+
def self.all
|
7
|
+
find_and_create
|
8
|
+
end
|
9
|
+
|
10
|
+
def initialize(table:, column:, suggestion:)
|
11
|
+
@table = table.to_sym
|
12
|
+
@column = column
|
13
|
+
@suggestion = suggestion
|
14
|
+
end
|
15
|
+
|
16
|
+
class << self
|
17
|
+
private
|
18
|
+
|
19
|
+
def find_and_create
|
20
|
+
relevant_tables.map do |table|
|
21
|
+
connection.columns(table).map do |column|
|
22
|
+
next if ignored_column?(table, column)
|
23
|
+
|
24
|
+
rec = recommended_comment(column)
|
25
|
+
rec ? new(table: table, column: column, suggestion: rec) : nil
|
26
|
+
end.compact
|
27
|
+
end.compact.flatten
|
28
|
+
end
|
29
|
+
|
30
|
+
def connection
|
31
|
+
ActiveRecord::Base.connection
|
32
|
+
end
|
33
|
+
|
34
|
+
def relevant_tables
|
35
|
+
connection.tables - PiiSafeSchema.configuration.ignore_tables
|
36
|
+
end
|
37
|
+
|
38
|
+
def ignored_column?(table, column)
|
39
|
+
PiiSafeSchema.configuration.
|
40
|
+
ignore_columns[table.to_sym]&.
|
41
|
+
include?(column.name.to_sym)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'pii_safe_schema/configuration'
|
2
|
+
require 'pii_safe_schema/annotations'
|
3
|
+
require 'pii_safe_schema/notify'
|
4
|
+
require 'pii_safe_schema/pii_column'
|
5
|
+
require 'pii_safe_schema/version'
|
6
|
+
require 'pii_safe_schema/notifiers/std_out'
|
7
|
+
require 'pii_safe_schema/notifiers/data_dog'
|
8
|
+
require 'pii_safe_schema/railtie'
|
9
|
+
require 'rails/generators'
|
10
|
+
require 'rails/generators/active_record/migration/migration_generator'
|
11
|
+
require 'pii_safe_schema/migration_generator'
|
12
|
+
require 'json'
|
13
|
+
|
14
|
+
module PiiSafeSchema
|
15
|
+
extend PiiSafeSchema::Notify
|
16
|
+
|
17
|
+
def self.configuration
|
18
|
+
@configuration ||= Configuration.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.configure
|
22
|
+
yield(configuration)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.activate!
|
26
|
+
return if Rails.env.test?
|
27
|
+
|
28
|
+
ActiveSupport.on_load :active_record do
|
29
|
+
Notify.notify(PiiSafeSchema::PiiColumn.all)
|
30
|
+
end
|
31
|
+
rescue ActiveRecord::NoDatabaseError
|
32
|
+
Rails.logger.info('PiiSafeSchema: No DB'.red)
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.generate_migrations
|
36
|
+
PiiSafeSchema::MigrationGenerator.generate_migrations(PiiSafeSchema::PiiColumn.all)
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'pii_safe_schema/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'pii_safe_schema'
|
7
|
+
s.version = PiiSafeSchema::VERSION
|
8
|
+
s.authors = ['Alexi Garrow']
|
9
|
+
s.email = ['agarrow@wealthsimple.com']
|
10
|
+
|
11
|
+
s.summary = 'Schema migration tool for checking and adding comments on PII columns.'
|
12
|
+
s.homepage = 'https://github.com/wealthsimple/pii-safe-schema'
|
13
|
+
|
14
|
+
s.files = `git ls-files -z`.split("\x0").reject do |f|
|
15
|
+
f.match(%r{^(test|spec|features)/})
|
16
|
+
end
|
17
|
+
|
18
|
+
s.bindir = 'exe'
|
19
|
+
s.executables = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
s.require_paths = ['lib']
|
21
|
+
|
22
|
+
s.add_dependency 'activesupport', '>= 5'
|
23
|
+
s.add_dependency 'colorize'
|
24
|
+
s.add_dependency 'rails', '>= 5'
|
25
|
+
|
26
|
+
s.add_development_dependency 'bundler', '~> 1.16'
|
27
|
+
s.add_development_dependency 'bundler-audit'
|
28
|
+
s.add_development_dependency 'dogstatsd-ruby'
|
29
|
+
s.add_development_dependency 'git'
|
30
|
+
s.add_development_dependency 'guard-rspec'
|
31
|
+
s.add_development_dependency 'pry'
|
32
|
+
s.add_development_dependency 'rake', '~> 10.0'
|
33
|
+
s.add_development_dependency 'rspec', '~> 3.0'
|
34
|
+
s.add_development_dependency 'rspec-collection_matchers'
|
35
|
+
s.add_development_dependency 'rspec-its'
|
36
|
+
s.add_development_dependency 'rubocop'
|
37
|
+
s.add_development_dependency 'simplecov'
|
38
|
+
s.add_development_dependency 'ws-style'
|
39
|
+
|
40
|
+
# Required by activerecord-safer_migrations
|
41
|
+
s.add_development_dependency 'pg', '~> 0.21'
|
42
|
+
s.add_development_dependency 'strong_migrations'
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,318 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pii_safe_schema
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alexi Garrow
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-04-16 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: colorize
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rails
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '5'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '5'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: bundler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.16'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.16'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler-audit
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: dogstatsd-ruby
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: git
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: guard-rspec
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: rake
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '10.0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '10.0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rspec
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '3.0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '3.0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rspec-collection_matchers
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec-its
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: rubocop
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
209
|
+
- !ruby/object:Gem::Dependency
|
210
|
+
name: simplecov
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
212
|
+
requirements:
|
213
|
+
- - ">="
|
214
|
+
- !ruby/object:Gem::Version
|
215
|
+
version: '0'
|
216
|
+
type: :development
|
217
|
+
prerelease: false
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - ">="
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: '0'
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: ws-style
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - ">="
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: '0'
|
237
|
+
- !ruby/object:Gem::Dependency
|
238
|
+
name: pg
|
239
|
+
requirement: !ruby/object:Gem::Requirement
|
240
|
+
requirements:
|
241
|
+
- - "~>"
|
242
|
+
- !ruby/object:Gem::Version
|
243
|
+
version: '0.21'
|
244
|
+
type: :development
|
245
|
+
prerelease: false
|
246
|
+
version_requirements: !ruby/object:Gem::Requirement
|
247
|
+
requirements:
|
248
|
+
- - "~>"
|
249
|
+
- !ruby/object:Gem::Version
|
250
|
+
version: '0.21'
|
251
|
+
- !ruby/object:Gem::Dependency
|
252
|
+
name: strong_migrations
|
253
|
+
requirement: !ruby/object:Gem::Requirement
|
254
|
+
requirements:
|
255
|
+
- - ">="
|
256
|
+
- !ruby/object:Gem::Version
|
257
|
+
version: '0'
|
258
|
+
type: :development
|
259
|
+
prerelease: false
|
260
|
+
version_requirements: !ruby/object:Gem::Requirement
|
261
|
+
requirements:
|
262
|
+
- - ">="
|
263
|
+
- !ruby/object:Gem::Version
|
264
|
+
version: '0'
|
265
|
+
description:
|
266
|
+
email:
|
267
|
+
- agarrow@wealthsimple.com
|
268
|
+
executables: []
|
269
|
+
extensions: []
|
270
|
+
extra_rdoc_files: []
|
271
|
+
files:
|
272
|
+
- ".circleci/config.yml"
|
273
|
+
- ".github/CODEOWNERS"
|
274
|
+
- ".github/PULL_REQUEST_TEMPLATE.md"
|
275
|
+
- ".gitignore"
|
276
|
+
- ".rspec"
|
277
|
+
- ".rubocop.yml"
|
278
|
+
- ".ruby-version"
|
279
|
+
- CHANGELOG.md
|
280
|
+
- Gemfile
|
281
|
+
- Guardfile
|
282
|
+
- README.md
|
283
|
+
- Rakefile
|
284
|
+
- lib/pii_safe_schema.rb
|
285
|
+
- lib/pii_safe_schema/annotations.rb
|
286
|
+
- lib/pii_safe_schema/configuration.rb
|
287
|
+
- lib/pii_safe_schema/migration_generator.rb
|
288
|
+
- lib/pii_safe_schema/notifiers/data_dog.rb
|
289
|
+
- lib/pii_safe_schema/notifiers/std_out.rb
|
290
|
+
- lib/pii_safe_schema/notify.rb
|
291
|
+
- lib/pii_safe_schema/pii_column.rb
|
292
|
+
- lib/pii_safe_schema/railtie.rb
|
293
|
+
- lib/pii_safe_schema/version.rb
|
294
|
+
- lib/tasks/pii_safe_schema.rake
|
295
|
+
- pii-safe-schema.gemspec
|
296
|
+
homepage: https://github.com/wealthsimple/pii-safe-schema
|
297
|
+
licenses: []
|
298
|
+
metadata: {}
|
299
|
+
post_install_message:
|
300
|
+
rdoc_options: []
|
301
|
+
require_paths:
|
302
|
+
- lib
|
303
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
304
|
+
requirements:
|
305
|
+
- - ">="
|
306
|
+
- !ruby/object:Gem::Version
|
307
|
+
version: '0'
|
308
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
309
|
+
requirements:
|
310
|
+
- - ">="
|
311
|
+
- !ruby/object:Gem::Version
|
312
|
+
version: '0'
|
313
|
+
requirements: []
|
314
|
+
rubygems_version: 3.0.1
|
315
|
+
signing_key:
|
316
|
+
specification_version: 4
|
317
|
+
summary: Schema migration tool for checking and adding comments on PII columns.
|
318
|
+
test_files: []
|