pragmatic_tokenizer 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d9af93943342474b5d71cd2f57a08e1f6bbc51d
-  data.tar.gz: efcbcfd9f0c49dd74b6cd121780c45d3fe54dc57
+  metadata.gz: 9defa80663e9e5955967ba95d1bd346b5e06cca3
+  data.tar.gz: 5f3ec1a9392c2664b3ce8a26a7de9ba03e3a22e1
 SHA512:
-  metadata.gz: 13821d3deb7385c7b8ece7fe92e296a69318b0d8bf00a7284a0d52c0b38dcc2ecfa094bb90c21a04da2782adca30c726493f03ff5aa8f54c490897fb85e83e90
-  data.tar.gz: 871b3bf53dacc6ca2e7a9beaa62df5bb26101bb10ed1a84fe99acc6a8a8d7db33a01814982d0a60bdfd509f6eaac28b1ea358b499f833afa242dcd6f468c9cf6
+  metadata.gz: ddac620c5e335ee8ec3daa1c78ed96d9637e37d57dbb34886a153d7f292bad173ca823c291c44fc1a6f047276049d2e96562ab0a477c5cc58d6f0385636b3ba3
+  data.tar.gz: 9361da2cada5ab2e301601f4ddd50bde3fe7935f1b88447a3c5356b72063357a319471b4f1405f2c7aec3fa07f8c794f7a3aadc24a0e82919d2b829411725505

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -23,6 +23,7 @@ module PragmaticTokenizer
       shift_vertical_bar(text)
       convert_dbl_quotes(text)
       convert_sgl_quotes(text)
+      convert_apostrophe_s(text)
       shift_beginning_hyphen(text)
       shift_ending_hyphen(text)
       text.squeeze(' ')
@@ -120,6 +121,11 @@ module PragmaticTokenizer
       end
     end
 
+    def convert_apostrophe_s(text)
+      puts "Text: #{text.include?("\u{0301}")}"
+      text.gsub!(/\s\u{0301}(?=s(\s|\z))/, PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP['`']) || text
+    end
+
     def shift_beginning_hyphen(text)
       text.gsub!(/\s+-/, ' - ') || text
     end

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -122,7 +122,9 @@ module PragmaticTokenizer
     private
 
     def post_process(text)
+      puts "Text: #{text}"
       @tokens = PostProcessor.new(text: text, abbreviations: abbreviations).post_process
+      puts "Tokens: #{@tokens}"
       downcase! if downcase
       expand_contractions!(contractions) if expand_contractions
       clean! if clean

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Kevin S. Dias