pragmatic_tokenizer 3.0.7 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a0941801b61a33578d7d5dcc6ffe25e5ce524767
-  data.tar.gz: 611411f111648ec42d2cb23cf5c3be87e5a66510
+SHA256:
+  metadata.gz: 0feac0e3ed363c652e85fda4dd631b6f221da44af13ef9da24e9f0be5a0cdd38
+  data.tar.gz: adc17ceadcf5749c1c9d1abce01f28658511a9bd251e25dda2ccf333f32cd355
 SHA512:
-  metadata.gz: 4aaa482df0dcb555de46fe491d63da4ce70d222cb4b6392d1289d2d75e02e30039ccf1607c1b37c0741eb42616c051165ede5c3206eb05419da63fb7eda25564
-  data.tar.gz: fa08a0a58961213b94f327bd9e53af12b70797d7e6a41551aecf30e1ba10537efec0711077a1440c712dd707bce873b54707cdc3a30c6592421fc9a7d83518b4
+  metadata.gz: 99bbf93e63cf667703c2c386e51592873f9c70fa20df823d15ef69cf3714fef98c48d2428d451ce9c1433c3b1d23c5a317a2ecf1fc5ed6f9be04a66c8d268773
+  data.tar.gz: ddb68cee3ea6cf0b4d2bbe581c99a37d1ecb34de5a2c0703073cc53b27d6520dbf6d9bdd811f0bfcb244120c91290040f269eefeb002c3440d8c9fa55a2d9671

data/lib/pragmatic_tokenizer/languages/french.rb

@@ -11,11 +11,37 @@ module PragmaticTokenizer
         # why can't we directly reference constants from Languages::Common?
         ALNUM_QUOTE  = PragmaticTokenizer::Languages::Common::SingleQuotes::ALNUM_QUOTE
         QUOTE_WORD   = PragmaticTokenizer::Languages::Common::SingleQuotes::QUOTE_WORD
+        C_APOSTROPHE = /c'/i
+        J_APOSTROPHE = /j'/i
         L_APOSTROPHE = /l'/i
+        D_APOSTROPHE = /d'/i
+        QU_APOSTROPHE = /qu'/i
+        N_APOSTROPHE = /n'/i
+        T_APOSTROPHE = /t'/i
+        M_APOSTROPHE = /m'/i
+        S_APOSTROPHE = /s'/i
+        QUELQU_APOSTROPHE = /quelqu'/i
+        JUSQU_APOSTROPHE = /jusqu'/i
+        LORSQU_APOSTROPHE = /lorsqu'/i
+        PUISQU_APOSTROPHE = /puisqu'/i
+        QUOIQU_APOSTROPHE = /quoiqu'/i
 
         def handle_single_quotes(text)
           replacement = PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'".freeze]
+          text.gsub!(C_APOSTROPHE, '\1 c' << replacement << ' ')
+          text.gsub!(J_APOSTROPHE, '\1 j' << replacement << ' ')
           text.gsub!(L_APOSTROPHE, '\1 l' << replacement << ' ')
+          text.gsub!(D_APOSTROPHE, '\1 d' << replacement << ' ')
+          text.gsub!(QU_APOSTROPHE, '\1 qu' << replacement << ' ')
+          text.gsub!(N_APOSTROPHE, '\1 n' << replacement << ' ')
+          text.gsub!(T_APOSTROPHE, '\1 t' << replacement << ' ')
+          text.gsub!(M_APOSTROPHE, '\1 m' << replacement << ' ')
+          text.gsub!(S_APOSTROPHE, '\1 s' << replacement << ' ')
+          text.gsub!(QUELQU_APOSTROPHE, '\1 quelqu' << replacement << ' ')
+          text.gsub!(JUSQU_APOSTROPHE, '\1 jusqu' << replacement << ' ')
+          text.gsub!(LORSQU_APOSTROPHE, '\1 lorsqu' << replacement << ' ')
+          text.gsub!(PUISQU_APOSTROPHE, '\1 puisqu' << replacement << ' ')
+          text.gsub!(QUOIQU_APOSTROPHE, '\1 quoiqu' << replacement << ' ')
           text.gsub!(ALNUM_QUOTE,  '\1 '  << replacement << ' ')
           text.gsub!(QUOTE_WORD,   ' '    << replacement)
           text

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "3.0.7".freeze
+  VERSION = "3.1.0".freeze
 end

data/pragmatic_tokenizer.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "unicode"
   spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "stackprof"
   spec.add_development_dependency "rubocop"

data/spec/languages/french_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 describe PragmaticTokenizer do
   context 'Language: French (fr)' do
     it 'tokenizes a string #001' do
-      text = "L'art de l'univers, c'est un art"
+      text = "D'art de l'univers, c'est un art"
       pt = PragmaticTokenizer::Tokenizer.new(
           language: 'fr'
       )
-      expect(pt.tokenize(text)).to eq(["l'", "art", "de", "l'", "univers", ",", "c'est", "un", "art"])
+      expect(pt.tokenize(text)).to eq(["d'", "art", "de", "l'", "univers", ",", "c'" ,"est", "un", "art"])
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 3.0.7
+  version: 3.1.0
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-18 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode
@@ -42,16 +42,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -169,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A multilingual tokenizer