pragmatic_tokenizer 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 600855c6a883dea597e6abd87b43721579680fe2
-  data.tar.gz: f0dc608d45aac9701b6ded9b5de001cde3b987e9
+  metadata.gz: 46da26b8afc38bfc2699a2875d9786be47702160
+  data.tar.gz: 9073a0505244bb97d7fa51c48e9fe4ab30983c90
 SHA512:
-  metadata.gz: 93ac2871c7e053060289f5fd758a090470c5dc7a98a9d68453c965701e425c63ef0825b83715e081e3c69c2d31617efa1738fdca36411537e622b6409d975416
-  data.tar.gz: 2ec0fbdbdfdb29f91324853aea7ccd992ec4e3ca04274d97aead2eccc84b819b83e351e5554caac36211040508dfa01f0f8877c73a87a33c79bf5567f580f3f8
+  metadata.gz: 8397bca6ada5fae51d1894b26154b5f9fed73e375b9e78b2803da892c20e5f32efb2722f0fe767cbc171586a5a6979a720041de5f0fd5c50824a0498c55b8394
+  data.tar.gz: f58cae264490ce16bef8e5512760e992dd38a11ced8fa872ab5faff2e92f61e6414f7e9ffd732f1e2910de560b2e9a2e98e1d59aa7039426ee0fbf3d8daaaa1f

data/README.md

@@ -8,17 +8,16 @@ Pragmatic Tokenizer is a multilingual tokenizer to split a string into tokens.
 
 Add this line to your application's Gemfile:
 
-```ruby
-gem 'pragmatic_tokenizer'
+**Ruby**  
+```
+gem install pragmatic_tokenizer
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install pragmatic_tokenizer
+**Ruby on Rails**  
+Add this line to your application’s Gemfile:  
+```ruby 
+gem 'pragmatic_tokenizer'
+```
 
 ## Usage
 

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "1.3.1"
+  VERSION = "1.4.0"
 end

data/pragmatic_tokenizer.gemspec

@@ -13,9 +13,9 @@ Gem::Specification.new do |spec|
   spec.description   = %q{A multilingual tokenizer to split a string into tokens.}
   spec.homepage      = "https://github.com/diasks2/pragmatic_tokenizer"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "unicode"

data/spec/languages/bulgarian_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe PragmaticTokenizer do
+  context 'Language: Bulgarian (bg)' do
+    it 'tokenizes a string #001' do
+      text = 'Стойностни, вкл. български и руски'
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'bg'
+      )
+      expect(pt.tokenize).to eq(["стойностни", ",", "вкл.", "български", "и", "руски"])
+    end
+
+    it 'tokenizes a string #002' do
+      text = 'Той поставя началото на могъща династия, която управлява в продължение на 150 г. Саргон надделява в двубой с владетеля на град Ур и разширява териториите на държавата си по долното течение на Тигър и Ефрат.'
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'bg',
+        remove_stop_words: true
+      )
+      expect(pt.tokenize).to eq(["поставя", "началото", "могъща", "династия", ",", "управлява", "продължение", "150", "саргон", "надделява", "двубой", "владетеля", "град", "ур", "разширява", "териториите", "държавата", "долното", "течение", "тигър", "ефрат", "."])
+    end
+
+    it 'tokenizes a string #003' do
+      text = 'Без български жертви в Париж.'
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'bg',
+        remove_stop_words: true
+      )
+      expect(pt.tokenize).to eq(["български", "жертви", "париж", "."])
+    end
+
+    it 'tokenizes a string #004' do
+      text = 'Без български жертви в Париж.'
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'bg',
+        remove_stop_words: true,
+        downcase: false
+      )
+      expect(pt.tokenize).to eq(["български", "жертви", "Париж", "."])
+    end
+  end
+end

data/spec/languages/deutsch_spec.rb

@@ -0,0 +1,229 @@
+require 'spec_helper'
+
+describe PragmaticTokenizer do
+  context 'Language: German (de)' do
+    it 'tokenizes a string #001' do
+      text = 'Das steht auf S. 23, s. vorherige Anmerkung.'
+      expect(PragmaticTokenizer::Tokenizer.new(text, language: 'de').tokenize).to eq(['das', 'steht', 'auf', 's.', '23', ',', 's.', 'vorherige', 'anmerkung', '.'])
+    end
+
+    it 'tokenizes a string #002' do
+      text = 'Die größte Ausdehnung des Landes vom Westen nach Osten beträgt 650 km – von Nord nach Süd sind es 560 km. Unter den europäischen Staaten ist Weißrussland flächenmäßig an 13'
+      expect(PragmaticTokenizer::Tokenizer.new(text,
+        language: 'de',
+        downcase: false,
+        remove_stop_words: true,
+        punctuation: 'none',
+        numbers: :none
+      ).tokenize).to eq(["größte", "Ausdehnung", "Landes", "Westen", "Osten", "beträgt", "Nord", "Süd", "europäischen", "Staaten", "Weißrussland", "flächenmäßig"])
+    end
+
+    it 'tokenizes a string #003' do
+      text = 'Die weißrussischen offiziellen Stellen wie auch die deutsche Diplomatie verwenden in offiziellen deutschsprachigen Texten den Namen Belarus, um die Unterscheidung von Russland zu verdeutlichen.'
+      expect(PragmaticTokenizer::Tokenizer.new(text,
+        language: 'de',
+        downcase: false
+      ).tokenize).to eq(["Die", "weißrussischen", "offiziellen", "Stellen", "wie", "auch", "die", "deutsche", "Diplomatie", "verwenden", "in", "offiziellen", "deutschsprachigen", "Texten", "den", "Namen", "Belarus", ",", "um", "die", "Unterscheidung", "von", "Russland", "zu", "verdeutlichen", "."])
+    end
+
+    it 'tokenizes a string #004' do
+      text = 'der Kaffee-Ersatz'
+      expect(PragmaticTokenizer::Tokenizer.new(text,
+        language: 'de',
+        downcase: false
+      ).tokenize).to eq(['der', 'Kaffee-Ersatz'])
+    end
+
+    it 'tokenizes a string #005' do
+      text = "Charlie Hebdo backlash over 'racist' Alan Kurdi cartoon - https://t.co/J8N2ylVV3w"
+      expect(PragmaticTokenizer::Tokenizer.new(text,
+        language: 'de',
+      ).tokenize).to eq(["charlie", "hebdo", "backlash", "over", "'", "racist", "'", "alan", "kurdi", "cartoon", "-", "https://t.co/j8n2ylvv3w"])
+    end
+
+    it 'handles words with a slash 1' do
+      text = "We pay 3000 €/month"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["we", "pay", "3000", "€", "month"])
+    end
+
+    it 'handles words with a slash 2' do
+      text = "Ich frage mich, wieso er nicht Herr der Lage war/ist."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["ich", "frage", "mich", "wieso", "er", "nicht", "herr", "der", "lage", "war", "ist"])
+    end
+
+    it 'handles words with a slash 3' do
+      text = "Poison gas attack in Ghuta/Syria."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["poison", "gas", "attack", "in", "ghuta", "syria"])
+    end
+
+    it 'handles words with a question mark' do
+      text = "Essen á la carte?Man ist versucht…"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["essen", "á", "la", "carte", "man", "ist", "versucht"])
+    end
+
+    it 'handles apostrophes and quotes 3' do
+      text = "Die “Mitte der Gesellschaft” interessiert sich jetzt für “Feminismus”."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["die", "mitte", "der", "gesellschaft", "interessiert", "sich", "jetzt", "für", "feminismus"])
+    end
+
+    it 'handles mentions 1' do
+      text = "@RainerSteinke @_Sternchen_2015 1:0 für dich."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["@rainersteinke", "@_sternchen_2015", "1:0", "für", "dich"])
+    end
+
+    it 'handles mentions 2' do
+      text = "@LandauDaniel @AnthZeto @julianfranz @S_Beck19 Yep!"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["@landaudaniel", "@anthzeto", "@julianfranz", "@s_beck19", "yep"])
+    end
+
+    it 'handles old school emoticons 1' do
+      text = "du übertreibst maßlos :D"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        downcase: false,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["du", "übertreibst", "maßlos", ":D"])
+    end
+
+    it 'handles words with a symbol suffix' do
+      text = "hier ist ein Whirlpool versteckt^^"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["hier", "ist", "ein", "whirlpool", "versteckt"])
+    end
+
+    it 'handles hashtags 1' do
+      text = "„Was wir tun wird in diesem Land Leben retten“:#Obama"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["was", "wir", "tun", "wird", "in", "diesem", "land", "leben", "retten", "#obama"])
+    end
+
+    it 'handles numbers and words' do
+      text = "Air Force Once ist 18.270-mal abgehoben."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["air", "force", "once", "ist", "18.270-mal", "abgehoben"])
+    end
+
+    it 'maintains the german gender-neutrality form 2' do
+      text = "der/die Lehrer_in und seine/ihre Schüler_innen"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["der", "die", "lehrer_in", "und", "seine", "ihre", "schüler_innen"])
+    end
+
+    it 'handles contractions 1' do
+      text = "gibt's"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        expand_contractions: true,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["gibt", "es"])
+    end
+
+    it 'handles contractions 2' do
+      text = "gibt‘s schaut’s wenn＇s g›spür find´s"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        expand_contractions: true,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["gibt", "es", "schaut", "es", "wenn", "es", "gespür", "finde", "es"])
+    end
+
+    it 'removes English stopwords' do
+      text = "der/die Lehrer_in und seine/ihre Schüler_innen. This has some English."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        filter_languages: [:en],
+        remove_stop_words: true,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["der", "die", "lehrer_in", "und", "seine", "ihre", "schüler_innen", ".", "english", "."])
+    end
+
+     it 'removes English and German stopwords' do
+      text = "der/die Lehrer_in und seine/ihre Schüler_innen. This has some English."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        filter_languages: [:en, :de],
+        remove_stop_words: true,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["lehrer_in", "schüler_innen", ".", "english", "."])
+    end
+
+    it 'does not remove English stopwords' do
+      text = "der/die Lehrer_in und seine/ihre Schüler_innen. This has some English."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["der", "die", "lehrer_in", "und", "seine", "ihre", "schüler_innen", ".", "this", "has", "some", "english", "."])
+    end
+
+    # I don't know how to easily treat these forms, especially the most frequent form
+    # that attaches "Innen" (plural) or "In" (singular) (with a capital I) to a word.
+    it 'maintains the german gender-neutrality form 1' do
+      skip "NOT IMPLEMENTED"
+      text = "Wir brauchen eine/n erfahrene/n Informatiker/in."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["wir", "brauchen", "eine/n", "erfahrene/n", "informatiker/in"])
+    end
+
+    it 'handles apostrophes and quotes 4' do
+      skip "NOT IMPLEMENTED"
+      text = "Endlich regnet es ihm nicht mehr auf ́s Haupt!"
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        punctuation: 'none',
+        language: 'de'
+      )
+      expect(pt.tokenize).to eq(["endlich", "regnet", "es", "ihm", "nicht", "mehr", "auf́s", "haupt"])
+    end
+
+    it 'handles abrreviations for languages other than English' do
+      text = "Adj. Smith how are ü. today."
+      pt = PragmaticTokenizer::Tokenizer.new(text,
+        language: :de
+      )
+      expect(pt.tokenize).to eq(["adj", ".", "smith", "how", "are", "ü.", "today", "."])
+    end
+  end
+end