pragmatic_tokenizer 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93070f0374fe808125850e652cb0f05cdd262c3e
-  data.tar.gz: 0e69cfc518e9b734ab35e47d7eb0d656aaa22a76
+  metadata.gz: 7bfc15f5e19157e3d4f5aba7f781dc4ce11ab73a
+  data.tar.gz: 8416accd1749a6f51868bfc59060a7bc3f30c708
 SHA512:
-  metadata.gz: e96d74097ee19c69c03952341be97f425200ea056dabd0b36dfae4387df78ca1920e521aaad1c28aef0e3d07b35d68abd21b54d7e21fcfc110c873b1ee319449
-  data.tar.gz: 908a93161d227b7cdca4208157a339c12c075b416ceed0a59757a9227ebde77c1c87e9deae694ba2445cbd1a2dd00f8cb81a91051826de3b8596654272a4e9b4
+  metadata.gz: 4d3a59c05e2e526d619f70bfdcb12bc9bf5299897c4fff026883d5a23a849274fa9966d4cf47cff7aeb07639cf70a5ced4015e5974326b00f631c5815753088e
+  data.tar.gz: a82349e9152127ed195a800803e1ee828459e286a0910147d53e5331c8b16f66516521f4d53b6a6bc3b80afb8d6995f4ec1ff290d46d59d50728eb1f46391d19

data/lib/pragmatic_tokenizer/processor.rb

@@ -90,8 +90,8 @@ module PragmaticTokenizer
 
     def shift_colon(text)
       return text unless text.include?(':') &&
-        text.partition(':').last[0] !~ /\A\d+/ &&
-        text.partition(':').first[-1] !~ /\A\d+/
+        (text.partition(':').last[0] !~ /\A\d+/ ||
+        text.partition(':').first[-1] !~ /\A\d+/)
       # Ignore web addresses
       text.gsub!(/(?<=[http|https]):(?=\/\/)/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP[":"]) || text
       text.gsub!(/:/o, ' :') || text

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -53,10 +53,11 @@ module PragmaticTokenizer
           split_at_middle_period_1(
           split_at_middle_period_2(
           split_beginning_period(
+          split_at_plus_sign(
           shift_no_spaces_between_sentences(
           split_at_forward_slash(
             processor.new(language: language_module).process(text: segment)
-          ))))))))))))).reject { |t| t.empty? }
+          )))))))))))))).reject { |t| t.empty? }
       end
       tokens.flatten
     end
@@ -192,6 +193,10 @@ module PragmaticTokenizer
       tokens.flat_map { |t| t.include?("/") && t !~ /(http|https|www)(\.|:)/ ? t.gsub!(/\//, '\1 \2').split(' ').flatten : t }
     end
 
+    def split_at_plus_sign(tokens)
+      tokens.flat_map { |t| t.include?("+") ? t.gsub!(/\+/, '\1 \2').split(' ').flatten : t }
+    end
+
     def find_contractions(tokens)
       return tokens unless expand_contractions && language_module::CONTRACTIONS
       if downcase

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Kevin S. Dias