confidential_info_redactor 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdee56aab509c61bb5746e12b5649f9dad2ad3ce
-  data.tar.gz: 3e47c212345e9986a53bdc6d1d38fc1c5c974341
+  metadata.gz: fd49179c366c68c4563353b5f11b5a547086c1e5
+  data.tar.gz: 91155c7b65e5267084e049b5f6876f5287e3df31
 SHA512:
-  metadata.gz: f7ff4ea744c2b5ec7fc2927efa23f0c0894973bf01cd4a2a39d21ea8575c274ec6e93d0a7e18f6f23bc28d9581b26cbea1de40a2d182eee2fe0cfd3be1ff28b9
-  data.tar.gz: e0ed6e7ac500a44fe84393ae2554623f6571f52141cf4e588e02de796dc752f4b0cdbd9d85077cac3f2d3dd2d9c20df7ff31af50f4fa4ac59363b91319c2f6af
+  metadata.gz: b6b5797b7113e41ad1bda8e1991b6e41939bff11f4ac48b0d0785552d1859b0db0da97551553ef56e97589e93d0446c07f5d747e81fec87692bac8933df211f5
+  data.tar.gz: f04627d38fd8de849da500afed42194204b6baa2a5fae8505afecebb766bc8f9a425e2281c7be579257711319752e081fe02dc86a47dbc0fa7dc644c0a578c86

data/lib/confidential_info_redactor/extractor.rb

@@ -23,7 +23,15 @@ module ConfidentialInfoRedactor
       extracted_terms = []
       PragmaticSegmenter::Segmenter.new(text: text, language: language).segment.each do |segment|
         initial_extracted_terms = segment.gsub(EXTRACT_REGEX).map { |match| match unless corpus.include?(match.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/”/,'').gsub(/\'$/, '')) }.compact
-        next if initial_extracted_terms.length.eql?(segment.split(' ').length)
+        in_corpus = true
+        initial_extracted_terms.each do |ngram|
+          ngram.split(/[\?\)\(\!\\\/\"\:\;\,]/).each do |t|
+            unless corpus.include?(t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip)
+              in_corpus = false
+            end
+          end
+        end
+        next if initial_extracted_terms.length.eql?(segment.split(' ').length) && in_corpus
         initial_extracted_terms.each do |ngram|
           ngram.split(/[\?\)\(\!\\\/\"\:\;\,]/).each do |t|
             next if !(t !~ /.*\d+.*/)

data/lib/confidential_info_redactor/version.rb

@@ -1,3 +1,3 @@
 module ConfidentialInfoRedactor
-  VERSION = "0.0.15"
+  VERSION = "0.0.16"
 end

data/spec/confidential_info_redactor/extractor_spec.rb

@@ -141,7 +141,7 @@ RSpec.describe ConfidentialInfoRedactor::Extractor do
 
       it 'extracts the proper nouns from a text #017' do
         text = 'John'
-        expect(described_class.new(text: text, language: 'en').extract).to eq([])
+        expect(described_class.new(text: text, language: 'en').extract).to eq(['John'])
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: confidential_info_redactor
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - Kevin S. Dias