pii_safe_schema 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e10cda981f36b6e67b9964dfbe011856436d48242844ed569dff0d6406607512
-  data.tar.gz: dfc71c971b632a0cf124a2bf42a494cd1306e6b4a7178b4d3de085f45a6931cf
+  metadata.gz: 5a3fe6f6dc02bc520f523874f6111cb00dc9201687f0b7f668dc39e587080bda
+  data.tar.gz: c2f2bf99f36d65c806e9b74af88ab55a8c77291e5bd61883a36bd2b2bf2d6b1b
 SHA512:
-  metadata.gz: 0e1c90b37fa8c3a70522bacb74b172855135878af97a355403e79325d8ed32a849f173015fcec844e5f3213715bacc5ba4ce18380550f667b92f9126940f821f
-  data.tar.gz: c8044fb8a4d0065ed97625c93e04a7e82dad3ab338d6a878e0fb5839ea26b7d1b7b9f8fa5fddca8413010ddd46306bb4235314f242575ce698c3d8c9e66cbac9
+  metadata.gz: 539795ee77529477a46a52df234f0474f5d774f2ded25537520f20d023af8539c5643f3c859e3df56e3181591cea9b854b23030be1d5f4c9344dec6b3b7d7f29
+  data.tar.gz: 8ed9a5f69eeeca205128085268198732888632563922be5402129779074f314df3bf18ad54b17a859372fb90c2689a2b140352bd02f9fa9275950b75667deb67

data/.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 defaults: &defaults
   working_directory: /home/circleci/wealthsimple
   docker:
-    - image: circleci/ruby:2.6.0
+    - image: circleci/ruby:2.6.5
     - image: circleci/postgres:9.5.9-alpine
       environment:
         POSTGRES_USER: circleci

data/.ruby-version

@@ -1 +1 @@
-2.6.0
+2.6.5

data/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.3.0 - 2019-11-04
+### Added
+- Can pass explicitly annotate PII columns from the command line as arguments when using `rake pii_safe_schema:generate_migrations`.
+
 ## 1.2.0 - 2019-4-20
 ### Added
 - Can pass Datadog Client object as a configuration option.

data/README.md

@@ -46,7 +46,7 @@ PiiSafeSchema.configure do |config|
     some_table:       :*,                       # ignore the whole table
     some_other_table: [:column_1, :column_2]    # just those columns
   }
-  
+
   # Pass whatever instance you want here, but it must implement the method
   # #event(title, message, opts = {})
   # which is what datadog-statsd does:
@@ -60,12 +60,21 @@ end
 
 ## Generating Comment Migrations
 
-```ruby
+```bash
 rake pii_safe_schema:generate_migrations
 ```
 
-This will generate one migration file for each table that should be commented.
-it will create a comment field for each column that it warns you about when you start a rails server or console.
+This will generate one migration file for each table that should be commented. It will create a comment field for each column that it warns you about when you start a rails server or console.
+
+### Explicit annotations
+
+If the generator fails to identify a PII column, you can specify explicitly what columns in what tables are PII. This is particularly useful if you're installed pii_safe_schema into an existing project.
+
+```bash
+rake pii_safe_schema:generate_migrations [table:column:annotation_type] ...
+```
+
+Run `rake pii_safe_schema:generate_migrations help` for details
 
 ## Credits
 

data/lib/pii_safe_schema.rb

@@ -1,3 +1,4 @@
+require 'pii_safe_schema/invalid_column_error'
 require 'pii_safe_schema/configuration'
 require 'pii_safe_schema/annotations'
 require 'pii_safe_schema/notify'
@@ -36,7 +37,56 @@ module PiiSafeSchema
     Rails.logger.info('PiiSafeSchema: No DB'.red)
   end
 
-  def self.generate_migrations
-    PiiSafeSchema::MigrationGenerator.generate_migrations(PiiSafeSchema::PiiColumn.all)
+  def self.generate_migrations(additional_pii_columns = [])
+    PiiSafeSchema::MigrationGenerator.generate_migrations(
+      PiiSafeSchema::PiiColumn.all + additional_pii_columns,
+    )
+  end
+
+  def self.parse_additional_columns(arguments)
+    arguments.map do |str|
+      matches = /([a-z_]+):([a-z_]+):([a-z_]+)/i.match(str)
+      return print_help! if matches.blank?
+
+      suggestion = Annotations.comment(matches[3])
+      return print_help! if suggestion.blank?
+
+      PiiColumn.from_column_name(table: matches[1], column: matches[2], suggestion: suggestion)
+    end
+  end
+
+  def self.print_help!(do_exit: true) # rubocop:disable Metrics/MethodLength
+    puts <<~HELPMSG # rubocop:disable Rails/Output
+      Usage:
+        rake pii_safe_schema:generate_migrations [table:column:annotation_type] ...
+
+      Arguments:
+        [table:column:annotation_type]   # A column to manually annotate. Can be repeated.
+                                         # annotation_type can be "email", "phone", "ip_address",
+                                         # "geolocation", "address", "postal_code", "name",
+                                         # "sensitive_data", or "encrypted_data"
+
+      Description:
+        Generates a migration to add PII annotation comments to appropriate columns on a table.
+        Uses a series of regular expressions to find sensitive fields.
+
+        Optionally supply arguments to annotate columns explicitly
+
+      Example:
+        rake pii_safe_schema:generate_migrations signatures:signatory_name:name signatures:landline:phone
+
+        Will generate a migration with the following, assuming automatic regex had no matches:
+
+        class ChangeCommentsInSignatures < ActiveRecord::Migration[5.2]
+          def change
+            safety_assured do
+              change_column :signatures, :signatory_name, :string, comment: '{"pii":{"obfuscate":"name_obfuscator"}}'
+              change_column :signatures, :landline, :string, comment: '{"pii":{"obfuscate":"phone_obfuscator"}}'
+            end
+          end
+        end
+    HELPMSG
+
+    exit(1) if do_exit # rubocop:disable Rails/Exit
   end
 end

data/lib/pii_safe_schema/annotations.rb

@@ -70,14 +70,18 @@ module PiiSafeSchema
       nil
     end
 
+    def self.comment(annotation_type)
+      COLUMNS.dig(annotation_type.to_sym, :comment)
+    end
+
     def apply_recommendation?(column, pii_info)
       !encrypted?(column) &&
-        pii_info[:regexp].match(column.name) &&
+        pii_info[:regexp].match?(column.name) &&
         column.comment != pii_info[:comment].to_json
     end
 
     def encrypted?(column)
-      COLUMNS[:encrypted_data][:regexp].match(column.name)
+      COLUMNS[:encrypted_data][:regexp].match?(column.name)
     end
 
     def apply_encrypted_recommendation?(column)

data/lib/pii_safe_schema/invalid_column_error.rb

@@ -0,0 +1,4 @@
+module PiiSafeSchema
+  class InvalidColumnError < StandardError
+  end
+end

data/lib/pii_safe_schema/pii_column.rb

@@ -3,10 +3,6 @@ module PiiSafeSchema
     extend PiiSafeSchema::Annotations
     attr_reader :table, :column, :suggestion
 
-    def self.all
-      find_and_create
-    end
-
     def initialize(table:, column:, suggestion:)
       @table = table.to_sym
       @column = column
@@ -14,6 +10,18 @@ module PiiSafeSchema
     end
 
     class << self
+      def all
+        find_and_create
+      end
+
+      def from_column_name(table:, column:, suggestion:)
+        unless connection.columns(table.to_s).find { |c| c.name == column.to_s }
+          raise InvalidColumnError, "column \"#{column}\" does not exist for table \"#{table}\""
+        end
+
+        new(table: table, column: column, suggestion: suggestion)
+      end
+
       private
 
       def find_and_create

data/lib/pii_safe_schema/version.rb

@@ -1,3 +1,3 @@
 module PiiSafeSchema
-  VERSION = '1.2.0'.freeze
+  VERSION = '1.3.0'.freeze
 end

data/lib/tasks/pii_safe_schema.rake

@@ -1,5 +1,25 @@
 namespace :pii_safe_schema do
   task generate_migrations: :environment do
-    PiiSafeSchema.generate_migrations
+    PiiSafeSchema.print_help! if ARGV[2] == 'help'
+
+    if ARGV.length == 1
+      PiiSafeSchema.generate_migrations
+    else
+      additional_columns = PiiSafeSchema.parse_additional_columns(ARGV[1..])
+      PiiSafeSchema.generate_migrations(additional_columns)
+    end
+
+  rescue ActiveRecord::StatementInvalid, PiiSafeSchema::InvalidColumnError => e
+    raise e if e.class == ActiveRecord::StatementInvalid && e.cause.class != PG::UndefinedTable
+
+    puts <<~HEREDOC
+      Unable to generate PII annotation migration. Either the underlying table or column does not exist:
+
+      #{e.message}
+
+      Please create the table & columns first, running their migrations, before attempting to use the pii_safe_schema generator.
+    HEREDOC
+  ensure
+    exit(0) # forces rake to stop after this and not assume args are tasks
   end
 end

data/pii_safe_schema.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |s|
   s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
   s.require_paths = ['lib']
 
-  s.add_dependency 'activesupport', '>= 5'
+  s.add_dependency 'activesupport', '>= 5', '< 7'
   s.add_dependency 'colorize'
-  s.add_dependency 'rails', '>= 5'
+  s.add_dependency 'rails', '>= 5', '< 7'
 
   s.add_development_dependency 'bundler', '>= 1.16'
   s.add_development_dependency 'bundler-audit'
@@ -30,13 +30,14 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'git'
   s.add_development_dependency 'guard-rspec'
   s.add_development_dependency 'pry'
+  s.add_development_dependency 'rails', '>= 5.2.3', '< 7'
   s.add_development_dependency 'rake', '>= 10.0'
   s.add_development_dependency 'rspec', '< 4', '>= 3.0'
   s.add_development_dependency 'rspec-collection_matchers'
   s.add_development_dependency 'rspec-its'
   s.add_development_dependency 'rubocop'
   s.add_development_dependency 'simplecov'
-  s.add_development_dependency 'sqlite3-ruby'
+  s.add_development_dependency 'sqlite3'
   s.add_development_dependency 'ws-style'
 
   # Required by activerecord-safer_migrations

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pii_safe_schema
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Alexi Garrow
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-20 00:00:00.000000000 Z
+date: 2019-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -17,6 +17,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -24,6 +27,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -45,6 +51,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -52,6 +61,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +162,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.2.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -241,7 +273,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3-ruby
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -320,6 +352,7 @@ files:
 - lib/pii_safe_schema.rb
 - lib/pii_safe_schema/annotations.rb
 - lib/pii_safe_schema/configuration.rb
+- lib/pii_safe_schema/invalid_column_error.rb
 - lib/pii_safe_schema/migration_generator.rb
 - lib/pii_safe_schema/notifiers/data_dog.rb
 - lib/pii_safe_schema/notifiers/std_out.rb
@@ -347,7 +380,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Schema migration tool for checking and adding comments on PII columns.