stockade 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84a7317f812734960f8ed6f56c0194d783aad1c7
-  data.tar.gz: 6d361b573054fb0b0b2c5d19a02cebe7cb705173
+  metadata.gz: 2e40900217ae6ce9c9548707578efcfe90d27211
+  data.tar.gz: 8130b7c181f4d8ff0b2e322d4a408b4311505aa8
 SHA512:
-  metadata.gz: 431ecb92de2cd3e67596af6347d23c09e86bc4efbd8436617bdd599d25140fa67872385b848bd989635b24f8477da44d880047c2ac112fa8614f9b291b844727
-  data.tar.gz: 0b9040960eb148c06f4a664d6f074c075200afd36f10046c8c50eaa914d5550ebb1c91ea8ca5f9d53d0add1f0ddfb464c7b0636beb5ec6e361eaeefce129f417
+  metadata.gz: 39b25eff37a4405e38312d5c6cce3836468f1db7791421f90176060140496c27dfec84cd5e78289e2a316e00787884a09c6d18dbe9681ca7a851693ddd76f1d5
+  data.tar.gz: abf9b314b14437d08dc1fc5116a51d2d129881097ac6491d0ac66c6c73cc16af9f0749af0b8b1886a45126fecdbf790ce27751063e2fcf9eeed0fb2051d6cb9b

data/Gemfile.lock

@@ -2,19 +2,35 @@ PATH
   remote: .
   specs:
     stockade (0.1.1)
+      credit_card_validations (~> 3.5)
       memoist (~> 0.1)
-      rambling-trie
+      rambling-trie (~> 2.0)
+      rubyzip (~> 1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activemodel (5.2.1)
+      activesupport (= 5.2.1)
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
     ast (2.4.0)
     byebug (10.0.2)
     coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    credit_card_validations (3.5.0)
+      activemodel (>= 3, <= 6)
+      activesupport (>= 3, <= 6)
     diff-lcs (1.3)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.1)
     memoist (0.16.0)
     method_source (0.9.0)
+    minitest (5.11.3)
     parallel (1.12.1)
     parser (2.5.1.2)
       ast (~> 2.4.0)
@@ -50,6 +66,10 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.9.0)
+    rubyzip (1.2.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
     unicode-display_width (1.4.0)
 
 PLATFORMS
@@ -57,10 +77,10 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.16)
-  pry-byebug
+  pry-byebug (~> 3.0)
   rake (~> 10.0)
   rspec (~> 3.0)
-  rubocop
+  rubocop (~> 0.49)
   stockade!
 
 BUNDLED WITH

data/README.md

@@ -1,11 +1,10 @@
-# PII Lexer
+# PII Detector
 
 _This is a proof-of-concept level software._
 
-Stockade is a lexer for Personally Identifiable Information (PII). It scans
-unstructured text (from files, logs, databases, web etc.) and tokenized
-recognized pieces of PII. This information can be used to raise errors,
-discard, mask data.
+Stockade is a Personally Identifiable Information (PII) detector. It scans
+unstructured text (from files, logs, databases, web etc.) and masks all
+identified pieces of PII.
 
 ## Installation
 
@@ -17,21 +16,40 @@ gem install stockade
 
 ```ruby
 require 'stockade'
-#=> true
-Stockade.mask('Mr. John Smith email is jsmith@example.com')
-#=> "Mr. **** ***** email is ******************"
-Stockade.mask('and his phone is 555-123-4567.')
-#=> *** his phone is ************.
+
+puts Stockade.mask(<<-EOS
+
+Dossier on Mr. John Smith born 09/02/1995
+His email is jsmith@example.com and his phone is 555-123-4567.
+He is using Visa card 4111 1111 1111 1111
+
+EOS
+
+#=>
+Dossier on Mr. **** ***** born **********
+His email is ****************** and his phone is ************.
+** is using Visa card *******************
+
 ```
-Yes, 'and' looks like PII because there is also a lastname.
+Notice, how word 'He' was incorrectly identified as a name.
 
 ## Implementation
 
-It uses
+This is done in three stages.
+
+### Scanning
+
+Using a manually curated list of regexes and
 [StringScanner](https://ruby-doc.org/stdlib-2.5.1/libdoc/strscan/rdoc/StringScanner.html)
-and a manually curated list of regular expressions to match strings that _look_
-like PII. This works for things like emails, phone numbers, dates, national
-ids, credit card numbers and ip addresses. But it does not work for names.
-Names are verified against the list of known first and last names that are
-stored as a trie.
+it extracts and labels lexeme candidates.
+
+### Evaluation
+
+Lexeme candidates further evaluated (in some cases this is a no-op) to filter
+out false positives.  For example, first and lastnames are checked against a
+database of known names. Dates are checked to be in the past.
+
+### Parsing
 
+Some rudimentary parsing done. Lexemes that are fully covered by other lexemes
+are eliminated. Ambiguous lexemes are disambiguated using rules of precedence.

data/bin/load

@@ -8,5 +8,5 @@ require 'rambling-trie'
 
 %w[lastnames firstnames words].each do |type|
   trie = Rambling::Trie.create("data/#{type}.txt")
-  Rambling::Trie.dump(trie, "data/#{type}.dump")
+  Rambling::Trie.dump(trie, "data/#{type}.zip")
 end

data/data/phones.txt

@@ -0,0 +1,13 @@
+#-###-###-####
++#-###-###-####
++##-###-###-####
++###-###-###-####
+###-###-####
+### ### ####
+(## ##) #### ####
+##########
+(##) #### ####
+(##) ## #### ####
+###-###-###-####
+###-####
+(###) ###-####

data/lib/stockade.rb

@@ -13,17 +13,41 @@ require 'stockade/lexemes/dict'
 require 'stockade/lexemes/word'
 require 'stockade/lexemes/lastname'
 require 'stockade/lexemes/firstname'
+require 'stockade/lexemes/payment_card'
 
 # Stockade module
 module Stockade
-  # Mask all PII in `text` with `*`
-  #
-  def self.mask(text)
-    lexemes = Parser.call(Lexer.call(text))
-    lexemes.inject(text) do |mask, lexeme|
-      prefix = lexeme.start_pos.zero? ? '' : mask[0..lexeme.start_pos - 1]
-      postfix = mask[lexeme.end_pos..-1]
-      "#{prefix}#{lexeme.mask}#{postfix}"
+  class << self
+    # Mask all PII in `text` with `*`
+    #
+    def mask(text)
+      process(text, :mask)
+    end
+
+    def tokenize(text)
+      process(text, :token)
+    end
+
+    def process(text, action)
+      lexemes(text).inject(text) do |mask, lexeme|
+        prefix = lexeme.start_pos.zero? ? '' : mask[0..lexeme.start_pos - 1]
+        postfix = mask[lexeme.end_pos..-1]
+        "#{prefix}#{lexeme.send(action)}#{postfix}"
+      end
+    end
+
+    def extract(text)
+      lexemes(text).map do |lexeme|
+        {
+          lexeme.class.name.to_s.split('::').last.downcase => lexeme.value
+        }
+      end
+    end
+
+    private
+
+    def lexemes(text)
+      Parser.call(Lexer.call(text))
     end
   end
 end

data/lib/stockade/lexemes/base.rb

@@ -42,6 +42,24 @@ module Stockade
       def mask
         '*' * raw_value.size
       end
+
+      def token
+        SecureRandom.base64(raw_value.size)[0..raw_value.size - 1]
+      end
+
+      def type
+        self.class.name.split('::').last.downcase.to_sym
+      end
+
+      def self.types
+        %i[date word email firstname lastname phone]
+      end
+
+      types.each do |type_name|
+        define_method :"#{type_name}?" do
+          type == type_name
+        end
+      end
     end
   end
 end

data/lib/stockade/lexemes/dict.rb

@@ -27,13 +27,17 @@ module Stockade
         Word.new(value: value).valid?
       end
 
+      def capitalized?
+        raw_value[0] == raw_value[0].upcase
+      end
+
       class << self
         extend Memoist
 
         def dict_name; end
 
         def dict
-          Rambling::Trie.load("data/#{dict_name}.dump")
+          Rambling::Trie.load("data/#{dict_name}.zip")
         end
         memoize :dict
       end

data/lib/stockade/lexemes/filler.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    class Filler < Base
+    end
+  end
+end

data/lib/stockade/lexemes/payment_card.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'credit_card_validations'
+
+module Stockade
+  module Lexemes
+    # Lexeme for anything that resembles payment card numbers
+    # https://en.wikipedia.org/wiki/Payment_card_number
+    #
+    # Any 10-19 character long sequences of digits optionally grouped using
+    # ' ' or '-' delimiters are suspects
+    class PaymentCard < Base
+      def self.regex
+        /
+          (?<!\d) # NaN
+          \d
+          ([\s\-]*\d[\s\-]*){10,17}
+          \d
+          (?!\d)  # NaN
+        /x
+      end
+
+      def valid?
+        CreditCardValidations::Detector.new(value).valid?
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/phone.rb

@@ -4,22 +4,7 @@ module Stockade
   module Lexemes
     # Phone lexeme
     class Phone < Base
-      # Less noisy phone mask syntax compared to regexes
-      MASKS = [
-        '#-###-###-####',
-        '+#-###-###-####',
-        '+##-###-###-####',
-        '+###-###-###-####',
-        '###-###-####',
-        '### ### ####',
-        '(## ##) #### ####',
-        '##########',
-        '(##) #### ####',
-        '(##) ## #### ####',
-        '###-###-###-####',
-        '###-####',
-        '(###) ###-####'
-      ].freeze
+      MASKS = File.readlines('data/phones.txt').freeze
 
       class << self
         def regex
@@ -33,7 +18,7 @@ module Stockade
 
         private
 
-        # Convert phone number mask to its regex
+        # Convert less noisy phone mask syntax to regexes
         # ### ### #### => (?:\d{3}\s\d{3}\s\d{4})
         def to_re(mask)
           '(?:' +

data/lib/stockade/lexemes/word.rb

@@ -12,6 +12,10 @@ module Stockade
       def mask
         raw_value
       end
+
+      def token
+        raw_value
+      end
     end
   end
 end

data/lib/stockade/lexer.rb

@@ -27,6 +27,7 @@ module Stockade
 
     def lexeme_classes
       [
+        Stockade::Lexemes::PaymentCard,
         Stockade::Lexemes::Date,
         Stockade::Lexemes::Email,
         Stockade::Lexemes::Phone,

data/lib/stockade/parser.rb

@@ -75,18 +75,13 @@ module Stockade
     end
 
     def priority(lexeme)
-      lexeme_priorities.index(lexeme.class)
-    end
-
-    def lexeme_priorities
-      [
-        Lexemes::Word,
-        Lexemes::Firstname,
-        Lexemes::Lastname,
-        Lexemes::Phone,
-        Lexemes::Date,
-        Lexemes::Email
-      ]
+      if [Lexemes::Firstname, Lexemes::Lastname].include?(lexeme.class)
+        lexeme.capitalized? ? 3 : 1
+      elsif lexeme.is_a?(Lexemes::Word)
+        2
+      else
+        3
+      end
     end
   end
 end

data/lib/stockade/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Stockade
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/stockade.gemspec

@@ -4,6 +4,7 @@ lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'stockade/version'
 
+# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name          = 'stockade'
   spec.version       = Stockade::VERSION
@@ -29,8 +30,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug', '~> 3.0'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.5'
+  spec.add_development_dependency 'rubocop', '~> 0.49'
 
+  spec.add_runtime_dependency 'credit_card_validations', '~> 3.5'
   spec.add_runtime_dependency 'memoist', '~> 0.1'
   spec.add_runtime_dependency 'rambling-trie', '~> 2.0'
+  spec.add_runtime_dependency 'rubyzip', '~> 1.0'
 end
+# rubocop:enable Metrics/BlockLength

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockade
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Stan Mazhara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-05 00:00:00.000000000 Z
+date: 2018-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,14 +72,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.49'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.49'
+- !ruby/object:Gem::Dependency
+  name: credit_card_validations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: memoist
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: "\n    Stockade is a lexer that reads unstructured text information (from
   files,\n    logs, databases etc.) and tokenizes pieces that look like personally\n
   \   identifiable information (PII).\n  "
@@ -124,19 +152,19 @@ files:
 - README.md
 - Rakefile
 - bin/load
-- data/firstnames.dump
-- data/firstnames.txt
-- data/lastnames.dump
-- data/lastnames.txt
-- data/words.dump
-- data/words.txt
+- data/firstnames.zip
+- data/lastnames.zip
+- data/phones.txt
+- data/words.zip
 - lib/stockade.rb
 - lib/stockade/lexemes/base.rb
 - lib/stockade/lexemes/date.rb
 - lib/stockade/lexemes/dict.rb
 - lib/stockade/lexemes/email.rb
+- lib/stockade/lexemes/filler.rb
 - lib/stockade/lexemes/firstname.rb
 - lib/stockade/lexemes/lastname.rb
+- lib/stockade/lexemes/payment_card.rb
 - lib/stockade/lexemes/phone.rb
 - lib/stockade/lexemes/word.rb
 - lib/stockade/lexer.rb