nlp-pure 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7adc921eb51b54bd646cc0c7c57edbfd47a4d7fb
-  data.tar.gz: 54930de1e0cd9f5507bde731e057f22942f1d212
+  metadata.gz: 8ae3951baabcafe913e157a575e3dc718a646f16
+  data.tar.gz: 14a6567449629a482bdc8863ffbfd04ae72af61b
 SHA512:
-  metadata.gz: a0ee82f8e519e712d36f89af779e9b42aed4887e0b87fa5c77e6b422f5b8ab6d66afc702c650f3bd9149a80d7c2fbb1434504fb53e132164c5dafb8c520b5102
-  data.tar.gz: db7bf9e6178c2e25a5d5bcdde763dbeb8fc9d3130800a0bfd41a8a6abae1ab64ad012ca27bd7334c241e8c4486566cd6f60a8f9c173086c53747f3296d7e9fed
+  metadata.gz: f1766d42dd2916bdb0491448a9db0122b86f31325e3f12ce94c0d6b403cf5ecf50e4e95139f018f76896c1ef432e71a11dc36d8c2c597dc0870f400fb56bfeae
+  data.tar.gz: b3baa2f16339813070ffa978e03e8972046baebba815259d695da517baedc2830a55d0cfa75dcd234e307d2f09684194c02aa83a3b919f3db08be1426eb71537

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.0.5
+
+Fixed bug in `NlpPure::Segmenting::DefaultWord` where leading ellipses could produce extra segmented words.
+
 # 0.0.4
 
 Fixed bug in `NlpPure::Segmenting::DefaultWord` where ellipses without spaces would not segment.

data/lib/nlp_pure/segmenting/default_word.rb

@@ -7,13 +7,29 @@ module NlpPure
       DEFAULT_OPTIONS = {
         # 3+ periods as pseudo-ellipsis (with optional whitespace)
         # OR hyphen, en dash, em dash, and whitespace
-        split: /\s?\.{3,}+\s?|[\s\-–—…]+/
+        split: /\s?\.{3,}\s?|[\s\-–—…]+/,
+        # array of arrays; [0] should be regexp, [1] should be replacement
+        # NOTE: minor performance risk in letting this array grow long
+        gsub:  [
+          # ellipses at the start of a string are problematic; ref #12
+          [/^\s?(…|\.{3,})/, ' ']
+        ]
       }.freeze
+
       def self.parse(*args)
         unless args.nil? || args.empty?
-          input = args[0].to_s
-          input.split(options[:split])
+          clean_input(args[0]).split(options[:split])
+        end
+      end
+
+      def self.clean_input(text = nil)
+        input = text.to_s
+        # perform replacements to work around the limitations of the splitting regexp
+        options.fetch(:gsub, []).each do |gsub_pair|
+          input.gsub!(gsub_pair[0], gsub_pair[1])
         end
+        # NOTE: leading whitespace is problematic; ref #12
+        input.strip
       end
 
       # NOTE: exposed as a method for easy mock/stub

data/lib/nlp_pure/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 #
 module NlpPure
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/spec/lib/segmenting/default_word_spec.rb

@@ -18,9 +18,11 @@ describe NlpPure::Segmenting::DefaultWord do
       let(:english_twohyphen_sentence) { 'The quick brown fox--full of energy--jumps over the lazy dog.' }
       let(:english_ellipsis_sentence) { 'The quick brown fox…jumps over the lazy dog.' }
       let(:english_spaced_ellipsis_sentence) { 'The quick brown fox … jumps over the lazy dog.' }
-      let(:english_period_ellipses_sentence) { 'The quick brown fox...jumps over the lazy dog.' }
-      let(:english_trailing_ellipses_sentence) { 'The quick brown fox jumps over the lazy dog …' }
-      let(:english_spaced_period_ellipses_sentence) { 'The quick brown fox ... jumps over the lazy dog.' }
+      let(:english_period_ellipsis_sentence) { 'The quick brown fox...jumps over the lazy dog.' }
+      let(:english_leading_ellipsis_sentence) { ' … the quick brown fox jumps over the lazy dog.' }
+      let(:english_leading_period_ellipsis_sentence) { ' ... the quick brown fox jumps over the lazy dog.' }
+      let(:english_trailing_ellipsis_sentence) { 'The quick brown fox jumps over the lazy dog … ' }
+      let(:english_spaced_period_ellipsis_sentence) { 'The quick brown fox ... jumps over the lazy dog.' }
       let(:english_abbreviation_sentence) { 'The U.S.A. is a member of NATO.' }
       let(:english_simple_paragraph) { 'Mary had a little lamb. The lamb’s fleece was white as snow. Everywhere that Mary went, the lamb was sure to go.' }
       let(:english_simple_line_breaks) { "Mary had a little lamb,\nHis fleece was white as snow,\nAnd everywhere that Mary went,\nThe lamb was sure to go." }
@@ -79,15 +81,19 @@ describe NlpPure::Segmenting::DefaultWord do
         end
 
         it 'correctly segments period-ellipses' do
-          expect(NlpPure::Segmenting::DefaultWord.parse(english_period_ellipses_sentence).length).to eq(9)
+          expect(NlpPure::Segmenting::DefaultWord.parse(english_period_ellipsis_sentence).length).to eq(9)
         end
 
         it 'correctly segments spaced period-ellipses' do
-          expect(NlpPure::Segmenting::DefaultWord.parse(english_spaced_period_ellipses_sentence).length).to eq(9)
+          expect(NlpPure::Segmenting::DefaultWord.parse(english_spaced_period_ellipsis_sentence).length).to eq(9)
+        end
+
+        it 'correctly segments with leading, spaced ellipses' do
+          expect(NlpPure::Segmenting::DefaultWord.parse(english_leading_ellipsis_sentence).length).to eq(9)
         end
 
         it 'correctly segments with trailing, spaced ellipses' do
-          expect(NlpPure::Segmenting::DefaultWord.parse(english_trailing_ellipses_sentence).length).to eq(9)
+          expect(NlpPure::Segmenting::DefaultWord.parse(english_trailing_ellipsis_sentence).length).to eq(9)
         end
 
         it 'does not segment abbreviations' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nlp-pure
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Reid Parham