stockade 0.1.0 → 0.1.1

data/lib/stockade/lexemes/base.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Base class for all lexemes
+    #
+    # Lexer extracts lexem candidates of text using `.regex` of
+    # corresponding= subclass, instantiates it and then furtner calls
+    # its `#valid?` to verify that this is indeed a valid lexeme.
+    #
+    class Base
+      attr_reader :raw_value, :start_pos
+
+      def initialize(value, start_pos = nil)
+        @raw_value = value
+        @start_pos = start_pos
+      end
+
+      def value
+        raw_value.downcase.strip
+      end
+
+      def end_pos
+        start_pos + raw_value.size
+      end
+
+      def self.regex; end
+
+      def valid?
+        true
+      end
+
+      def ==(other)
+        value == other.value &&
+          self.class == other.class
+      end
+
+      def range
+        start_pos..end_pos
+      end
+
+      def mask
+        '*' * raw_value.size
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/date.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Date lexeme
+    class Date < Base
+      class << self
+        def regex
+          /
+          (?<!\d)
+           (\d{1,4})
+            #{delim}
+          (\d{1,4})
+            #{delim}
+          (\d{1,4})
+          (?!\d)
+          /x
+        end
+
+        def delim
+          %r{[\s\.\-\/]}
+        end
+      end
+
+      def valid?
+        possible_dates.any? &&
+          possible_dates.all? do |date|
+            date <= ::Date.today
+          end
+      end
+
+      private
+
+      def possible_dates
+        parts.permutation.map do |permutation|
+          begin
+            ::Date.new(*permutation)
+          rescue ArgumentError
+            nil
+          end
+        end.compact
+      end
+
+      def parts
+        self.class.regex.match(value).captures.map(&:to_i)
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/dict.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Abstract Dictionary lexeme
+    #
+    # Dictionary lexemes are those that can only be verified by
+    # checking the corresponding dictionary
+    #
+    class Dict < Base
+      def self.regex
+        /
+          [a-zA-Z]+
+        /x
+      end
+
+      def valid?
+        return false unless self.class.dict
+        self.class.dict.include?(value)
+      end
+
+      def name
+        raise 'Abstract'
+      end
+
+      def common_word?
+        Word.new(value: value).valid?
+      end
+
+      class << self
+        extend Memoist
+
+        def dict_name; end
+
+        def dict
+          Rambling::Trie.load("data/#{dict_name}.dump")
+        end
+        memoize :dict
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/email.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Email lexeme
+    class Email < Base
+      def self.regex
+        /
+        [\w+\-\.\+]+
+        @
+        [a-z\d\-]+
+        (\.[a-z]+)*
+        \.[a-z]+ # TLD
+        /x
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/firstname.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Firstname lexeme
+    class Firstname < Dict
+      class << self
+        def dict_name
+          'firstnames'
+        end
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/lastname.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Lastname lexeme
+    class Lastname < Dict
+      class << self
+        def dict_name
+          'lastnames'
+        end
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/phone.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # Phone lexeme
+    class Phone < Base
+      # Less noisy phone mask syntax compared to regexes
+      MASKS = [
+        '#-###-###-####',
+        '+#-###-###-####',
+        '+##-###-###-####',
+        '+###-###-###-####',
+        '###-###-####',
+        '### ### ####',
+        '(## ##) #### ####',
+        '##########',
+        '(##) #### ####',
+        '(##) ## #### ####',
+        '###-###-###-####',
+        '###-####',
+        '(###) ###-####'
+      ].freeze
+
+      class << self
+        def regex
+          /
+          #{MASKS
+            .map { |mask| to_re(mask) }
+            .join(" |\n")
+          }
+          /x
+        end
+
+        private
+
+        # Convert phone number mask to its regex
+        # ### ### #### => (?:\d{3}\s\d{3}\s\d{4})
+        def to_re(mask)
+          '(?:' +
+            mask
+            .gsub('+', '\\\+')
+            .gsub(/(#+)/) { |m| "\\d{#{m.size}}" }
+            .gsub(' ', '\s')
+            .gsub('(', '\(\s*')
+            .gsub(')', '\\s*\)') +
+            ')'
+        end
+      end
+    end
+  end
+end

data/lib/stockade/lexemes/word.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Stockade
+  module Lexemes
+    # A word found is common words dictionary
+    class Word < Dict
+      def self.dict_name
+        'words'
+      end
+
+      # common dictionary words are safe
+      def mask
+        raw_value
+      end
+    end
+  end
+end

data/lib/stockade/lexer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'rambling-trie'
+require 'memoist'
+require 'strscan'
+require 'pry-byebug'
+
+module Stockade
+  # Class Lexer
+  #
+  # Usage `Stockade::Lexer.call(context)`
+  #
+  # Returns list of found lexemes.
+  #
+  class Lexer
+    extend Memoist
+
+    attr_reader :context
+
+    def initialize(context)
+      @context = context.dup
+    end
+
+    def self.call(context)
+      new(context).call
+    end
+
+    def lexeme_classes
+      [
+        Stockade::Lexemes::Date,
+        Stockade::Lexemes::Email,
+        Stockade::Lexemes::Phone,
+        Stockade::Lexemes::Word,
+        Stockade::Lexemes::Firstname,
+        Stockade::Lexemes::Lastname
+      ]
+    end
+
+    def call
+      lexeme_classes.map do |lexeme_class|
+        tokenize(lexeme_class)
+      end.flatten
+    end
+
+    private
+
+    def tokenize(lexeme_class)
+      lexemes = []
+      scanner = StringScanner.new(context)
+
+      while scanner.scan_until(lexeme_class.regex)
+        lexemes << lexeme_class.new(
+          scanner.matched,
+          scanner.pos - scanner.matched.size
+        )
+      end
+
+      lexemes.select(&:valid?)
+    end
+  end
+end

data/lib/stockade/parser.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Stockade
+  # Parser
+  #
+  # Takes a raw list (a set really) of lexemes returned by Lexer
+  # and further processes them. Currently this means:
+  # 1. Deduping - >1 ambiguous lexemes matching exactly same fragment
+  #    are deduped according to their priority. Only the highest priority
+  #    lexeme kept.
+  # 2. Removing covered lexems - lexeme that matches a string that is a
+  # substring of another matched string is removed.
+  #
+  class Parser
+    extend Memoist
+
+    attr_reader :lexemes
+
+    def initialize(lexemes)
+      @lexemes = lexemes
+    end
+
+    def self.call(lexemes)
+      new(lexemes).call
+    end
+
+    def call
+      non_covered_lexemes
+    end
+
+    private
+
+    def grouped
+      lexemes.group_by(&:range).values
+    end
+    memoize :grouped
+
+    def deduped
+      grouped.map do |group|
+        if group.size == 1
+          group.first
+        else
+          group.max_by do |lexeme|
+            priority(lexeme)
+          end
+        end
+      end
+    end
+    memoize :deduped
+
+    def ordered
+      lexemes.sort_by(&:start_pos)
+    end
+    memoize :ordered
+
+    def non_covered_lexemes
+      res = deduped.dup
+
+      res.each_index do |index|
+        head, *rest = *res[index..-1]
+
+        rest.each do |lex|
+          next unless covers?(head, lex)
+          res.delete(lex)
+        end
+      end.to_a
+
+      res
+    end
+
+    def covers?(head, lex)
+      head != lex &&
+        head.start_pos <= lex.start_pos &&
+        head.end_pos >= lex.end_pos
+    end
+
+    def priority(lexeme)
+      lexeme_priorities.index(lexeme.class)
+    end
+
+    def lexeme_priorities
+      [
+        Lexemes::Word,
+        Lexemes::Firstname,
+        Lexemes::Lastname,
+        Lexemes::Phone,
+        Lexemes::Date,
+        Lexemes::Email
+      ]
+    end
+  end
+end

data/lib/stockade/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Stockade
-  VERSION = "0.1.0"
+  VERSION = '0.1.1'
 end

data/lib/stockade.rb

@@ -1,114 +1,29 @@
-require 'stockade/version'
+# frozen_string_literal: true
 
-require 'bloomfilter-rb'
 require 'memoist'
-require 'strscan'
 
+require 'stockade/version'
+require 'stockade/lexer'
+require 'stockade/parser'
+require 'stockade/lexemes/base'
+require 'stockade/lexemes/date'
+require 'stockade/lexemes/email'
+require 'stockade/lexemes/phone'
+require 'stockade/lexemes/dict'
+require 'stockade/lexemes/word'
+require 'stockade/lexemes/lastname'
+require 'stockade/lexemes/firstname'
+
+# Stockade module
 module Stockade
-  class Lexer
-    extend Memoist
-
-    attr_reader :datum
-
-    def initialize(datum)
-      @datum = datum.strip.dup
-    end
-
-    def self.call(datum)
-      new(datum).call
-    end
-
-    # order is important - from most specific to least
-    # the first one that matches stops the scan
-    def patterns
-      {
-        email: email_regex,
-        phone: phone_regex,
-        name: name_regex,
-      }
-    end
-
-    def scanner
-      StringScanner.new(datum)
-    end
-    memoize :scanner
-
-    def call
-      res = []
-
-      patterns.each do |name, regex|
-        scanner = StringScanner.new(datum)
-
-        loop do
-          break unless scanner.scan_until(regex)
-          value = scanner.matched
-
-          lexeme = name
-          if lexeme == :name
-            lexeme = :surname if surname?(value)
-            lexeme = :firstname if firstname?(value)
-          end
-          next if lexeme == :name
-
-          res << {
-            lexeme: lexeme,
-            value: scanner.matched
-          }
-
-          @datum = @datum[0..scanner.pos-scanner.matched.size] +
-            '*' * scanner.matched.size +
-            @datum[scanner.pos..-1]
-        end
-      end
-
-      res
-    end
-
-    def name_regex
-      /\w+/
-    end
-
-    def word_regex
-      /\W+/
-    end
-
-    private def email_regex
-      /
-      [\w+\-\.\+]+
-        @
-        [a-z\d\-]+
-        (\.[a-z]+)*
-        \.[a-z]+ # TLD
-        /x
-    end
-
-    private def email_address?
-      datum =~ email_regex
-    end
-
-    private def phone_number?
-      datum =~ phone_number_regex
-    end
-
-    private def phone_regex
-      /\b(?:\+?(\d{1,3}))?[-. (]*(\d{3})[-. )]*(\d{3})[-. ]*(\d{4})(?: *x(\d+))?\b/
-    end
-
-    private def surname?(value)
-      found?('surnames', value)
-    end
-
-    private def firstname?(value)
-      found?('firstnames', value)
-    end
-
-    private def found?(db, value)
-      db(db).include?(value.downcase)
-    end
-
-    private def db(name)
-      Marshal.load(File.read("data/#{name}.dump"))
+  # Mask all PII in `text` with `*`
+  #
+  def self.mask(text)
+    lexemes = Parser.call(Lexer.call(text))
+    lexemes.inject(text) do |mask, lexeme|
+      prefix = lexeme.start_pos.zero? ? '' : mask[0..lexeme.start_pos - 1]
+      postfix = mask[lexeme.end_pos..-1]
+      "#{prefix}#{lexeme.mask}#{postfix}"
     end
-    memoize :db
   end
 end

data/stockade.gemspec

@@ -1,4 +1,6 @@
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'stockade/version'
 
@@ -8,23 +10,27 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Stan Mazhara']
   spec.email         = ['akmegran@gmail.com']
 
-  spec.summary       = %q{Stockade is a lexer for PII}
-  spec.description   = %q{
+  spec.summary       = 'Stockade is a lexer for PII'
+  spec.description   = '
     Stockade is a lexer that reads unstructured text information (from files,
     logs, databases etc.) and tokenizes pieces that look like personally
     identifiable information (PII).
-  }
+  '
   spec.homepage      = 'https://github.com/smazhara/stockade'
   spec.license       = 'MIT'
 
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files`.split(/\n/).reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files`.split(/\n/).reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
 
   spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'pry-byebug', '~> 3.0'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.5'
 
-  spec.add_runtime_dependency 'bloomfilter-rb', '~> 2.0'
   spec.add_runtime_dependency 'memoist', '~> 0.1'
+  spec.add_runtime_dependency 'rambling-trie', '~> 2.0'
 end