stockade 0.1.0

data/lib/stockade/version.rb

@@ -0,0 +1,3 @@
+module Stockade
+  VERSION = "0.1.0"
+end

data/lib/stockade.rb

@@ -0,0 +1,114 @@
+require 'stockade/version'
+
+require 'bloomfilter-rb'
+require 'memoist'
+require 'strscan'
+
+module Stockade
+  class Lexer
+    extend Memoist
+
+    attr_reader :datum
+
+    def initialize(datum)
+      @datum = datum.strip.dup
+    end
+
+    def self.call(datum)
+      new(datum).call
+    end
+
+    # order is important - from most specific to least
+    # the first one that matches stops the scan
+    def patterns
+      {
+        email: email_regex,
+        phone: phone_regex,
+        name: name_regex,
+      }
+    end
+
+    def scanner
+      StringScanner.new(datum)
+    end
+    memoize :scanner
+
+    def call
+      res = []
+
+      patterns.each do |name, regex|
+        scanner = StringScanner.new(datum)
+
+        loop do
+          break unless scanner.scan_until(regex)
+          value = scanner.matched
+
+          lexeme = name
+          if lexeme == :name
+            lexeme = :surname if surname?(value)
+            lexeme = :firstname if firstname?(value)
+          end
+          next if lexeme == :name
+
+          res << {
+            lexeme: lexeme,
+            value: scanner.matched
+          }
+
+          @datum = @datum[0..scanner.pos-scanner.matched.size] +
+            '*' * scanner.matched.size +
+            @datum[scanner.pos..-1]
+        end
+      end
+
+      res
+    end
+
+    def name_regex
+      /\w+/
+    end
+
+    def word_regex
+      /\W+/
+    end
+
+    private def email_regex
+      /
+      [\w+\-\.\+]+
+        @
+        [a-z\d\-]+
+        (\.[a-z]+)*
+        \.[a-z]+ # TLD
+        /x
+    end
+
+    private def email_address?
+      datum =~ email_regex
+    end
+
+    private def phone_number?
+      datum =~ phone_number_regex
+    end
+
+    private def phone_regex
+      /\b(?:\+?(\d{1,3}))?[-. (]*(\d{3})[-. )]*(\d{3})[-. ]*(\d{4})(?: *x(\d+))?\b/
+    end
+
+    private def surname?(value)
+      found?('surnames', value)
+    end
+
+    private def firstname?(value)
+      found?('firstnames', value)
+    end
+
+    private def found?(db, value)
+      db(db).include?(value.downcase)
+    end
+
+    private def db(name)
+      Marshal.load(File.read("data/#{name}.dump"))
+    end
+    memoize :db
+  end
+end

data/stockade.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'stockade/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'stockade'
+  spec.version       = Stockade::VERSION
+  spec.authors       = ['Stan Mazhara']
+  spec.email         = ['akmegran@gmail.com']
+
+  spec.summary       = %q{Stockade is a lexer for PII}
+  spec.description   = %q{
+    Stockade is a lexer that reads unstructured text information (from files,
+    logs, databases etc.) and tokenizes pieces that look like personally
+    identifiable information (PII).
+  }
+  spec.homepage      = 'https://github.com/smazhara/stockade'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files`.split(/\n/).reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+
+  spec.add_runtime_dependency 'bloomfilter-rb', '~> 2.0'
+  spec.add_runtime_dependency 'memoist', '~> 0.1'
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: stockade
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Stan Mazhara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: memoist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: "\n    Stockade is a lexer that reads unstructured text information (from
+  files,\n    logs, databases etc.) and tokenizes pieces that look like personally\n
+  \   identifiable information (PII).\n  "
+email:
+- akmegran@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- Rakefile
+- bin/load
+- data/firstnames.dump
+- data/firstnames/1.csv
+- data/surnames.dump
+- data/surnames/1.csv
+- data/surnames/2.csv
+- lib/stockade.rb
+- lib/stockade/version.rb
+- stockade.gemspec
+homepage: https://github.com/smazhara/stockade
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Stockade is a lexer for PII
+test_files: []