pragmatic_tokenizer 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b2988beef12450f5a7f653ec5c2e8db5e11efce
-  data.tar.gz: 56d01b9c9ffef57ae3365f3519214318f7828a20
+  metadata.gz: f19baff424f25d8ba1fcf08001dfd6fc5ac4c54b
+  data.tar.gz: 645aaae817dd1a77c7fbfe506e5febe9746afbf7
 SHA512:
-  metadata.gz: 938592c183b4bd1f2fca41554d74e17b85f608549cfb218476c99c84829fec15e7ac6b2dbc4ce69b60a5780de90e0878203b5bfd54cdfbd71bfd0daed57d15f9
-  data.tar.gz: d767954ebcfd5003f1a58b0023307b93b22de30e08f491954511d1bcdafab4c3f23f5edb710f1d40515df98510bd63bcd9c427e4d2315f33b0d1641b726e998f
+  metadata.gz: c4a725af2141a66f0390258b36bc66a50994cc6f44e2db6b528b28e592df50baf1865d4445012dead0bf95a1347585acd2b93b22ba968ddf3a5fc08d4bcdaf79
+  data.tar.gz: 0a820f57952cdc0adb8973607b083bc0e14ba5006a7cf4ab5515f874c1a6f3bc5d1ce532b895c3011203ae74555480a7b81431906607ae26a376dcc5dd4a7d66

data/lib/pragmatic_tokenizer/languages/french.rb

@@ -11,8 +11,8 @@ module PragmaticTokenizer
           text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
           text.gsub!(/`(?!`)(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ') || text
           text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
-          text.gsub!(/l\'/, '\1 l\' \2') || text
-          text.gsub!(/L\'/, '\1 L\' \2') || text
+          text.gsub!(/l\'/, '\1 l☮ \2') || text
+          text.gsub!(/L\'/, '\1 L☮ \2') || text
         end
       end
     end

data/lib/pragmatic_tokenizer/processor.rb

@@ -22,7 +22,11 @@ module PragmaticTokenizer
       convert_sgl_quotes(text)
       shift_beginning_hyphen(text)
       shift_ending_hyphen(text)
-      tokens = separate_full_stop(text.squeeze(' ').split.map { |t| convert_sym_to_punct(t) })
+      tokens = separate_full_stop(text.squeeze(' ')
+        .split
+        .flat_map { |t| (t[0] == '‚' || t[0] == ',') && t.length > 1 ? t.split(/(,|‚)/).flatten : t }
+        .flat_map { |t| (t[-1] == '’' || t[-1] == "'") && t.length > 1 ? t.split(/(’|')/).flatten : t }
+        .map { |t| convert_sym_to_punct(t) })
       separate_other_ending_punc(tokens)
     end
 

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Kevin S. Dias