confidential_info_redactor 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4d140024c6ad95745f30e4ce6e9ecccce14bef8
-  data.tar.gz: 94510a7ae535f5b1d537cb64f0ddcb0cb76c8cef
+  metadata.gz: 2aba4fe814eb5d71124ad7a1b5f53840f1831d23
+  data.tar.gz: f682003e08e11ef4747255034fb40baed2fcf639
 SHA512:
-  metadata.gz: 5fb3c0593c24ce7e924da8906505040fc856d43bd6a422b15be892507c452cdc50dde8158f648942f911778a54447c204262def5914b4a728f541709519509c6
-  data.tar.gz: 8de04ce27bbeb9ae51c4793a52d518ccb4cf000b07f0ef893b54d8e099f11f8d08970a35be57cb4b445bb37b54a6dd1d53e7cfb36d189c4888d5e560189feeb8
+  metadata.gz: e39b8a938438e920cdfc2c158048dd779168fa16ff7a47ba29e5d7bc604c1d12f71ea626875aa805b83c12143a20fa256da33d594e332c279b99d5e049c07e99
+  data.tar.gz: 6b0096fe0d4b85068edc78326274d1a3d4a3b8bafe1f32bfe8d847623a9af65ff058d53dc004a28457e2eb33057065a62ce4277ef6b79ad83876d82c41334a47

data/lib/confidential_info_redactor/extractor.rb

@@ -22,19 +22,19 @@ module ConfidentialInfoRedactor
     def extract
       extracted_terms = []
       PragmaticSegmenter::Segmenter.new(text: text, language: language).segment.each do |segment|
-        initial_extracted_terms = segment.gsub(EXTRACT_REGEX).map { |match| match unless corpus.include?(match.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/\'$/, '')) }.compact
+        initial_extracted_terms = segment.gsub(EXTRACT_REGEX).map { |match| match unless corpus.include?(match.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/”/,'').gsub(/\'$/, '')) }.compact
         initial_extracted_terms.each do |ngram|
           ngram.split(/[\?\)\(\!\\\/\"\:\;\,]/).each do |t|
-            if corpus.include?(t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[0]) && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[0] != 'the' && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[0] != 'deutsche' && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ').length.eql?(2)
-              extracted_terms << t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[1] unless corpus.include?(t.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/\'$/, '').strip.split(' ')[1])
+            if corpus.include?(t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ')[0]) && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ')[0] != 'the' && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[0] != 'deutsche' && t.downcase.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ').length.eql?(2)
+              extracted_terms << t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ')[1] unless corpus.include?(t.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/\'$/, '').gsub(/”/,'').strip.split(' ')[1])
             else
               tracker = true
-              unless t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ').length.eql?(2) && t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ')[1].downcase.eql?('bank')
-                t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip.split(' ').each do |token|
+              unless t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ').length.eql?(2) && t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ')[1].downcase.eql?('bank')
+                t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip.split(' ').each do |token|
                   tracker = false if corpus.include?(token.downcase)
                 end
               end
-              extracted_terms << t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/\.\z/, '').strip unless corpus.include?(t.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/\'$/, '').strip) || !tracker
+              extracted_terms << t.gsub(/[\?\)\(\!\\\/\"\:\;\,]/, '').gsub(/\'$/, '').gsub(/”/,'').gsub(/\.\z/, '').strip unless corpus.include?(t.downcase.gsub(/[\?\.\)\(\!\\\/\"\:\;]/, '').gsub(/”/,'').gsub(/\'$/, '').strip) || !tracker
             end
           end
         end

data/lib/confidential_info_redactor/version.rb

@@ -1,3 +1,3 @@
 module ConfidentialInfoRedactor
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

data/spec/confidential_info_redactor/extractor_spec.rb

@@ -118,6 +118,21 @@ RSpec.describe ConfidentialInfoRedactor::Extractor do
         text = 'GOOD CARBS VS. BAD CARBS'
         expect(described_class.new(text: text, language: 'en').extract).to eq([])
       end
+
+      it 'extracts the proper nouns from a text #013' do
+        text = 'Reducing”'
+        expect(described_class.new(text: text, language: 'en').extract).to eq([])
+      end
+
+      it 'extracts the proper nouns from a text #014' do
+        text = '”'
+        expect(described_class.new(text: text, language: 'en').extract).to eq([])
+      end
+
+      it 'extracts the proper nouns from a text #015' do
+        text = '“Reducing'
+        expect(described_class.new(text: text, language: 'en').extract).to eq([])
+      end
     end
 
     context 'German (de)' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: confidential_info_redactor
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Kevin S. Dias