pii-detector 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4be466fefd667de37780c94c4af71fad5a310c89
+  data.tar.gz: 369b6ee78bd3a55a793a785e15ccaaf0310448c3
+SHA512:
+  metadata.gz: 3f55d0e0e71cee1d119cfd7ea72535245dd55ca2f4eb08832bd925b3075b356deb677295d709278093a91f1c3b3c314ecf6f91d18ef87e7053f02958823af538
+  data.tar.gz: af610219ad5697750fcd875aa1f1b9370320bb2bec440327780f373b4292594b62272383027abdfa1ed8b363e57472dbdc3e6d02fabb4f5048a6bea1fd638c3b

data/.gitignore

@@ -0,0 +1,4 @@
+.dev
+.bundle
+.rubocop-*
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_from:
+  - https://shopify.github.io/ruby-style-guide/rubocop.yml

data/Gemfile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'json'
+gem 'levenshtein-ffi'
+
+group :deployment do
+  gem 'package_cloud'
+  gem 'rake'
+end
+
+group :development do
+	gem 'rake'
+	gem 'pry'
+	gem 'minitest'
+	gem 'rubocop'
+end
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    pii-detector (1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    domain_name (0.5.20170404)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.9.18)
+    highline (1.6.20)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    json (2.1.0)
+    json_pure (1.8.1)
+    levenshtein-ffi (1.1.0)
+      ffi (~> 1.9)
+    method_source (0.9.0)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    minitest (5.11.3)
+    netrc (0.11.0)
+    package_cloud (0.3.03)
+      highline (= 1.6.20)
+      json_pure (= 1.8.1)
+      rainbow (= 2.2.2)
+      rest-client (~> 2.0)
+      thor (~> 0.18)
+    parallel (1.12.1)
+    parser (2.4.0.2)
+      ast (~> 2.3)
+    powerpack (0.1.1)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rainbow (2.2.2)
+      rake
+    rake (12.3.0)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rubocop (0.52.1)
+      parallel (~> 1.10)
+      parser (>= 2.4.0.2, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    thor (0.20.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.4)
+    unicode-display_width (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  json
+  levenshtein-ffi
+  minitest
+  package_cloud
+  pii-detector!
+  pry
+  rake
+  rubocop
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -0,0 +1 @@
+# PII Detector

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+task default: :test
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+end

data/bin/console

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('../../lib', __FILE__)
+
+require 'pii_detector'
+require 'pry'
+
+binding.pry

data/dev.yml

@@ -0,0 +1,21 @@
+name: pii-detector
+
+up:
+  - ruby: 2.5.0
+  - bundler
+
+commands:
+  test:
+    syntax:
+      argument: file
+      optional: args...
+    run: |
+      if [[ $# -eq 0 ]]; then
+        rake test
+      else
+        bundle exec ruby -Itest "$@"
+      fi
+  console:
+    run: bundle exec ruby bin/console
+  check:
+    run: cd lib/ && bundle exec rubocop

data/lib/config/pii_fields

@@ -0,0 +1,25 @@
+phone
+email
+full_name
+first_name
+last_name
+
+device_name
+user_agent
+
+zip
+address
+location
+latitude
+longitude
+
+ip
+gps
+
+sex
+gender
+
+token
+password
+
+religion

data/lib/hash.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'json'
+
+class Hash
+  def unique_keys
+    to_json.scan(/"([^"\\]+)"\s*:/).map(&:first).uniq
+  end
+end

data/lib/pii_detector.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'hash'
+
+require 'pii_detector/root'
+require 'pii_detector/version'
+require 'pii_detector/comparator'
+require 'pii_detector/configuration'
+
+module PIIDetector
+  extend self
+
+  # @param value [Hash]
+  # @return [Array]
+  def retrieve_pii_keys_from_hash(value)
+    value.unique_keys.map { |field| pii_field?(field) ? field : nil }.compact.uniq
+  end
+
+  # @param value [String]
+  # @return [Boolean]
+  def pii_field?(value)
+    config.pii_fields.any? { |pii| Comparator.similar?(pii, value) }
+  end
+
+  # @return [PIIDetector::Configuration]
+  def configure
+    @config ||= Configuration.new
+    yield(@config) if block_given?
+    @config
+  end
+
+  # @return [PIIDetector::Configuration]
+  def config
+    @config || configure
+  end
+end

data/lib/pii_detector/comparator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'levenshtein-ffi'
+require 'string'
+
+module PIIDetector
+  class Comparator
+    class << self
+      # @param desired [String]
+      # @param check [String]
+      # @return [Boolean] or [MatchData]
+      def similar?(desired, check)
+        standardized_desired_str = desired.strip.underscore
+        standardized_check_str = check.strip.underscore
+
+        return false if standardized_check_str.length <= 1
+
+        score = Levenshtein.distance(standardized_desired_str, standardized_check_str)
+
+        standardized_check_str == standardized_desired_str ||
+        standardized_check_str.match(/^#{standardized_desired_str}_/) ||
+        standardized_check_str.match(/_#{standardized_desired_str}?/) ||
+        standardized_check_str.match(/[a-z\d]*_#{standardized_desired_str}_[a-z\d]*/) ||
+        (PIIDetector.config.edit_distance < standardized_check_str.length && score < PIIDetector.config.edit_distance)
+      end
+    end
+  end
+end

data/lib/pii_detector/configuration.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module PIIDetector
+  class Configuration
+    attr_accessor :edit_distance
+    attr_reader :pii_fields
+
+    def initialize
+      @edit_distance = 2
+      @pii_fields = pii_fields_from_config
+    end
+
+    # @param value [Array] or [Set]
+    def pii_fields=(value)
+      @pii_fields = pii_fields_from_config.merge(value)
+    end
+
+    private
+
+    # @return [Set]
+    def pii_fields_from_config
+      @default_pii_fields ||=
+        File.read(File.join(PIIDetector.root_path, 'lib/config', 'pii_fields'))
+          .split("\n")
+          .reject(&:empty?)
+          .to_set
+    end
+  end
+end

data/lib/pii_detector/root.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module PIIDetector
+  # @return [String]
+  def root_path
+    File.expand_path('../../../', __FILE__)
+  end
+end

data/lib/pii_detector/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PIIDetector
+  VERSION = '1.0.0'
+end

data/lib/string.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class String
+  def underscore
+    gsub(/::/, '/')
+      .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+      .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+      .tr('-', '_')
+      .downcase
+  end
+end

data/pii_detector.gemspec

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require File.expand_path('../lib/pii_detector/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'pii-detector'
+  spec.summary       = 'Ruby gem to detect personally identifiable information.'
+  spec.version       = PIIDetector::VERSION
+  spec.authors       = ['']
+  spec.email         = ''
+  spec.homepage      = 'https://github.com/Shopify/pii-detector'
+  spec.license       = 'Shopify'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.require_paths = ['lib']
+end

data/shitip.yml

@@ -0,0 +1,13 @@
+deploy:
+  override:
+    - bundle exec rake build
+    - bundle exec package_cloud push shopify/gems pkg/*.gem
+
+dependencies:
+  override:
+    - bundle install
+  bundler:
+    without:
+      - development
+      - test
+      - debug

data/test/comparator_test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class ComparatorTest < Minitest::Test
+  def test_similar
+    assert PIIDetector::Comparator.similar?('first_name', 'first_name')
+    assert PIIDetector::Comparator.similar?('first_name', 'first_nam')
+    assert PIIDetector::Comparator.similar?('first_name', 'customer_first_name')
+    assert PIIDetector::Comparator.similar?('first_name', 'first_name_of_customer')
+    assert PIIDetector::Comparator.similar?('first_name', 'user_first_name_field')
+  end
+
+  def test_not_similar
+    refute PIIDetector::Comparator.similar?('first_name', 'event')
+    refute PIIDetector::Comparator.similar?('first_name', 'first')
+    refute PIIDetector::Comparator.similar?('first_name', 'first_na')
+    refute PIIDetector::Comparator.similar?('first_name', 'na')
+  end
+end

data/test/configuration_test.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class ConfigurationTest < Minitest::Test
+  def test_configuration_default_values
+    new_pii_fields = Set['unknown_field1']
+
+    PIIDetector.configure do |config|
+      config.edit_distance = 3
+      config.pii_fields += new_pii_fields
+    end
+
+    assert_equal 3, PIIDetector.config.edit_distance
+    assert new_pii_fields.subset?(PIIDetector.config.pii_fields)
+
+    new_pii_fields.merge(['unknown_field2'])
+    PIIDetector.config.edit_distance = 2
+    PIIDetector.config.pii_fields += new_pii_fields
+
+    assert_equal 2, PIIDetector.config.edit_distance
+    assert new_pii_fields.subset?(PIIDetector.config.pii_fields)
+  end
+
+  def test_configuration_set_not_exists_attribute
+    assert_raises NoMethodError do
+      PIIDetector.configure do |config|
+        config.unknown_attribute = 'test'
+      end
+    end
+  end
+
+  def test_configuration_get_not_exists_attribute
+    assert_raises NoMethodError do
+      PIIDetector.config.unknown_attribute
+    end
+  end
+end

data/test/hash_test.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class HashTest < Minitest::Test
+  def test_unique_keys_from_multilevel_hash
+    test_hash = {
+      a: {
+        b: {
+          c: 'c'
+        }
+      }
+    }
+
+    assert_equal ['a', 'b', 'c'], test_hash.unique_keys
+  end
+
+  def test_unique_keys_from_array_of_hashes
+    test_hash = {
+      a: [
+        { b: 'b' },
+        { c: 'c' }
+      ]
+    }
+
+    assert_equal ['a', 'b', 'c'], test_hash.unique_keys
+  end
+
+  def test_unique_keys_from_json_string_value
+    test_hash = {
+      a: {
+        b: "b"
+      }.to_json
+    }
+
+    assert_equal ['a'], test_hash.unique_keys
+  end
+end

data/test/pii_detector_test.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class PIIDetectorTest < Minitest::Test
+  def test_retrieve_pii_keys_from_hash
+    test_hash = {
+      first_name: 'Dave',
+      event: 'checkout'
+    }
+    assert_equal(['first_name'], PIIDetector.retrieve_pii_keys_from_hash(test_hash))
+  end
+
+  def test_pii_field
+    assert PIIDetector.pii_field?('first_name')
+    refute PIIDetector.pii_field?('event')
+  end
+end

data/test/string_test.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StringTest < Minitest::Test
+  def test_underscore
+    assert_equal 'first_name', 'FirstName'.underscore
+    assert_equal 'first_name', 'First_Name'.underscore
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'pii_detector'
+require 'minitest/autorun'
+
+# Configure PIIDetector first to escape warning
+PIIDetector.configure

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: pii-detector
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- ''
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-02-01 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- dev.yml
+- lib/config/pii_fields
+- lib/hash.rb
+- lib/pii_detector.rb
+- lib/pii_detector/comparator.rb
+- lib/pii_detector/configuration.rb
+- lib/pii_detector/root.rb
+- lib/pii_detector/version.rb
+- lib/string.rb
+- pii_detector.gemspec
+- shitip.yml
+- test/comparator_test.rb
+- test/configuration_test.rb
+- test/hash_test.rb
+- test/pii_detector_test.rb
+- test/string_test.rb
+- test/test_helper.rb
+homepage: https://github.com/Shopify/pii-detector
+licenses:
+- Shopify
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Ruby gem to detect personally identifiable information.
+test_files: []