stockade 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14344fa7658d3173f6f14e2300725ce2def9ff69
-  data.tar.gz: 2e5142b58d7a48dd811ad9bd2e5cfb228fea590d
+  metadata.gz: 84a7317f812734960f8ed6f56c0194d783aad1c7
+  data.tar.gz: 6d361b573054fb0b0b2c5d19a02cebe7cb705173
 SHA512:
-  metadata.gz: 8aa73b99f960c8fb42f2fba50a5406a0521ed5d78b77d9f02959a8aefcbdb0839e94d656cc6da98bd610b5fdd38812bc148555761db3f07735bc3a58917f8aa0
-  data.tar.gz: 0b622524d25184ae5313147ad83aa63d6e9e164b5b2b2f5c37f6131ccbc9a0d669cd473c877ced48afe5283d354de3f99220f22e38a19a3f2cbbb59636f988d5
+  metadata.gz: 431ecb92de2cd3e67596af6347d23c09e86bc4efbd8436617bdd599d25140fa67872385b848bd989635b24f8477da44d880047c2ac112fa8614f9b291b844727
+  data.tar.gz: 0b9040960eb148c06f4a664d6f074c075200afd36f10046c8c50eaa914d5550ebb1c91ea8ca5f9d53d0add1f0ddfb464c7b0636beb5ec6e361eaeefce129f417

data/Gemfile

@@ -1,4 +1,5 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
+source 'https://rubygems.org'
 
 gemspec

data/Gemfile.lock

@@ -1,28 +1,33 @@
 PATH
   remote: .
   specs:
-    stockade (0.1.0)
-      bloomfilter-rb
-      memoist
+    stockade (0.1.1)
+      memoist (~> 0.1)
+      rambling-trie
 
 GEM
   remote: https://rubygems.org/
   specs:
-    bloomfilter-rb (2.1.1)
-      redis
+    ast (2.4.0)
     byebug (10.0.2)
     coderay (1.1.2)
     diff-lcs (1.3)
+    jaro_winkler (1.5.1)
     memoist (0.16.0)
     method_source (0.9.0)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     pry-byebug (3.6.0)
       byebug (~> 10.0)
       pry (~> 0.10)
+    rainbow (3.0.0)
     rake (10.5.0)
-    redis (4.0.1)
+    rambling-trie (2.0.0)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
@@ -36,6 +41,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
+    rubocop (0.58.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    unicode-display_width (1.4.0)
 
 PLATFORMS
   ruby
@@ -45,6 +60,7 @@ DEPENDENCIES
   pry-byebug
   rake (~> 10.0)
   rspec (~> 3.0)
+  rubocop
   stockade!
 
 BUNDLED WITH

data/README.md

@@ -0,0 +1,37 @@
+# PII Lexer
+
+_This is a proof-of-concept level software._
+
+Stockade is a lexer for Personally Identifiable Information (PII). It scans
+unstructured text (from files, logs, databases, web etc.) and tokenized
+recognized pieces of PII. This information can be used to raise errors,
+discard, mask data.
+
+## Installation
+
+```
+gem install stockade
+```
+
+## Usage
+
+```ruby
+require 'stockade'
+#=> true
+Stockade.mask('Mr. John Smith email is jsmith@example.com')
+#=> "Mr. **** ***** email is ******************"
+Stockade.mask('and his phone is 555-123-4567.')
+#=> *** his phone is ************.
+```
+Yes, 'and' looks like PII because there is also a lastname.
+
+## Implementation
+
+It uses
+[StringScanner](https://ruby-doc.org/stdlib-2.5.1/libdoc/strscan/rdoc/StringScanner.html)
+and a manually curated list of regular expressions to match strings that _look_
+like PII. This works for things like emails, phone numbers, dates, national
+ids, credit card numbers and ip addresses. But it does not work for names.
+Names are verified against the list of known first and last names that are
+stored as a trie.
+

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/load

@@ -1,38 +1,12 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-# Make bloomfilter and dump it for future use
+# Convert plain text dictionaries to trie
 #
 require 'bundler/setup'
+require 'rambling-trie'
 
-require 'csv'
-require 'bloomfilter-rb'
-
-def load(type)
-  bf = BloomFilter::Native.new(
-    :size => 10_000_000,
-    :hashes => 2,
-    :seed => 1,
-    :bucket => 3,
-    :raise => false
-  )
-
-  Dir.glob("data/#{type}/*.csv").each do |file|
-    CSV.foreach(file) do |line|
-      name = line.first
-      next if name == 'name'
-
-      name.strip!
-      name.downcase!
-      bf.insert(name) unless bf.include?(name)
-    end
-  end
-
-  dump = Marshal.dump(bf)
-
-  File.write("data/#{type}.dump", dump)
-
-  df = Marshal.load(File.read("data/#{type}.dump"))
+%w[lastnames firstnames words].each do |type|
+  trie = Rambling::Trie.create("data/#{type}.txt")
+  Rambling::Trie.dump(trie, "data/#{type}.dump")
 end
-
-load('surnames')
-load('firstnames')