pragmatic_tokenizer 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e86a121879d806b58f855e311c14be249ba6ce95
-  data.tar.gz: 9be42b0a437ddaa0e03630d0fd6eee64f242bc9a
+  metadata.gz: c4834da7c6c1b1d6c614226840bb2fd5ef8b48b6
+  data.tar.gz: 395868d67e973b2a6e9e28b4b9883c95d1746fe6
 SHA512:
-  metadata.gz: 9f8fbf0b2de1674c557144568dc771fadcb882b892318eae04c7aae3f1ec53743f29dd208c971c45707ded50d88304aaa824ca2f5364fc65b96bd0b72d93e0d6
-  data.tar.gz: 39ee1f3e32cd243ef28c4f6b0823aa4cc523ca8a2adbb90cf6feca33fb966ccd452b70166f9ab7d2b64d859165b799c6ba3d45a410a2d5ff7116210170e77d02
+  metadata.gz: cc69a6f19545c9f5755df5c996e0625f0e65883fea81f01a877d10fce5f5b4eba8931529aecff9afb2ce56f8b993350d9bad15a94a5bb718db4eeafbbe611a29
+  data.tar.gz: f08442f148d59d98d3970e50ccc3bab2d59c1728fb06d9fefe4670ff5b4aca688168c81c30da3a49d1d800d8b398d53e76d9482c120450f2edc90b1b3c174617

data/README.md

@@ -26,36 +26,70 @@ Or install it yourself as:
 * To specify a language use its two character [ISO 639-1 code](https://www.tm-town.com/languages).
 * Pragmatic Tokenizer will unescape any HTML entities.
 
+**Example Usage**
+```ruby
+text = "\"I said, 'what're you? Crazy?'\" said Sandowsky. \"I can't afford to do that.\""
+
+PragmaticTokenizer::Tokenizer.new(text).tokenize
+# => ["\"", "i", "said", ",", "'", "what're", "you", "?", "crazy", "?", "'", "\"", "said", "sandowsky", ".", "\"", "i", "can't", "afford", "to", "do", "that", ".", "\""]
+
+# You can pass many different options:
+options = {
+  language:            :en, # the language of the string you are tokenizing
+  abbreviations:       ['a.b', 'a'], # a user-supplied array of abbreviations (downcased with ending period removed)
+  stop_words:          ['is', 'the'], # a user-supplied array of stop words (downcased)
+  remove_stop_words:   true, # remove stop words
+  contractions:        { "i'm" => "i am" }, # a user-supplied hash of contractions (key is the contracted form; value is the expanded form - both the key and value should be downcased)
+  expand_contractions: true, # (i.e. ["isn't"] will change to two tokens ["is", "not"])
+  filter_languages:    [:en, :de], # process abbreviations, contractions and stop words for this array of languages
+  punctuation:         :none, # see below for more details
+  numbers:             :none, # see below for more details
+  remove_emoji:        :true, # remove any emoji tokens
+  remove_urls:         :true, # remove any urls
+  remove_emails:       :true, # remove any emails
+  remove_domains:      :true, # remove any domains
+  hashtags:            :keep_and_clean, # remove the hastag prefix
+  mentions:            :keep_and_clean, # remove the @ prefix
+  clean:               true, # remove some special characters
+  classic_filter:      true, # removes dots from acronyms and 's from the end of tokens
+  downcase:            false, # do not downcase tokens
+  minimum_length:      3, # remove any tokens less than 3 characters
+  long_word_split:     10 # split tokens longer than 10 characters at hypens or underscores
+}
+```
+
 **Options**  
 
-##### `punctuation`
-  **default** = `'all'`
-- `'all'`  
-  Does not remove any punctuation from the result.
-- `'semi'`   
-  Removes full stops (i.e. periods) ['。', '．', '.'].
-- `'none'`  
-  Removes all punctuation from the result.
-- `'only'`  
-  Removes everything except punctuation. The returned result is an array of only the punctuation.
+##### `language`
+  **default** = `'en'`
+- To specify a language use its two character [ISO 639-1 code](https://www.tm-town.com/languages) as a symbol (i.e. `:en`) or string (i.e. `'en'`)
 
 <hr>
 
-##### `remove_stop_words`
-  **default** = `'false'`
-- `true`  
-  Removes all stop words.
-- `false`   
-  Does not remove stop words.
+##### `abbreviations`
+  **default** = `nil`
+- You can pass an array of abbreviations to overide or compliment the abbreviations that come stored in this gem. Each element of the array should be a downcased String with the ending period removed.
+
+<hr>
+
+##### `stop_words`
+  **default** = `nil`
+- You can pass an array of stop words to overide or compliment the stop words that come stored in this gem. Each element of the array should be a downcased String.
+
+<hr>
+
+##### `contractions`
+  **default** = `nil`
+- You can pass a hash of contractions to overide or compliment the contractions that come stored in this gem. Each key is the contracted form downcased and each value is the expanded form downcased.
 
 <hr>
 
-##### `remove_en_stop_words`
+##### `remove_stop_words`
   **default** = `'false'`
 - `true`  
-  Removes all English stop words (sometimes foreign language strings have English mixed in).
+  Removes all stop words.
 - `false`   
-  Does not remove English stop words.
+  Does not remove stop words.
 
 <hr>
 
@@ -68,180 +102,128 @@ Or install it yourself as:
 
 <hr>
 
-##### `clean`
+##### `filter_languages`
+  **default** = `nil`
+- You can pass an array of languages of which you would like to process abbreviations, stop words and contractions. This language can be indepedent of the language of the string you are tokenizing (for example your tex might be German but contain so English stop words that you want to remove). If you supply your own abbreviations, stop words or contractions they will be merged with the abbreviations, stop words and contractions of any languages you add in this option. You can pass an array of symbols or strings (i.e. `[:en, :de]` or `['en', 'de']`)
+
+<hr>
+
+##### `punctuation`
+  **default** = `'all'`
+- `:all`  
+  Does not remove any punctuation from the result.
+- `:semi`   
+  Removes full stops (i.e. periods) ['。', '．', '.'].
+- `:none`  
+  Removes all punctuation from the result.
+- `:only`  
+  Removes everything except punctuation. The returned result is an array of only the punctuation.
+
+<hr>
+
+##### `numbers`
+  **default** = `'all'`
+- `:all`  
+  Does not remove any numbers from the result
+- `:semi`   
+  Removes tokens that include only digits
+- `:none`  
+  Removes all tokens that include a number from the result (including Roman numerals)
+- `:only`  
+  Removes everything except tokens that include a number
+
+<hr>
+
+##### `remove_emoji`
   **default** = `'false'`
 - `true`  
-  Removes tokens consisting of only hypens, underscores, or periods as well as some special characters (®, ©, ™). Also removes long tokens or tokens with a backslash.
+  Removes any token that contains an emoji.
 - `false`   
   Leaves tokens as is.
 
 <hr>
 
-##### `remove_numbers`
+##### `remove_urls`
   **default** = `'false'`
 - `true`  
-  Removes any token that contains a number.
+  Removes any token that contains a URL.
 - `false`   
   Leaves tokens as is.
 
 <hr>
 
-##### `remove_roman_numerals`
+##### `remove_domains`
   **default** = `'false'`
 - `true`  
-  Removes any token that contains a Roman numeral.
+  Removes any token that contains a domain.
 - `false`   
   Leaves tokens as is.
 
 <hr>
 
-##### `downcase`
-  **default** = `'true'`
-
-<hr>
-
-##### `minimum_length`
-  **default** = `0`  
-  The minimum number of characters a token should be.  
-
-**Methods**  
-
-#### `#tokenize`
-
-**Example Usage**
-```ruby
-text = "\"I said, 'what're you? Crazy?'\" said Sandowsky. \"I can't afford to do that.\""
-
-PragmaticTokenizer::Tokenizer.new(text).tokenize
-# => ["\"", "i", "said", ",", "'", "what're", "you", "?", "crazy", "?", "'", "\"", "said", "sandowsky", ".", "\"", "i", "can't", "afford", "to", "do", "that", ".", "\""]
-
-PragmaticTokenizer::Tokenizer.new(text, remove_stop_words: true).tokenize
-# => ["\"", ",", "'", "what're", "?", "crazy", "?", "'", "\"", "sandowsky", ".", "\"", "afford", ".", "\""]
-
-PragmaticTokenizer::Tokenizer.new(text, punctuation: 'none').tokenize
-# => ["i", "said", "what're", "you", "crazy", "said", "sandowsky", "i", "can't", "afford", "to", "do", "that"]
-
-PragmaticTokenizer::Tokenizer.new(text, punctuation: 'only').tokenize
-# => ["\"", ",", "'", "?", "?", "'", "\"", ".", "\"", ".", "\""]
-
-PragmaticTokenizer::Tokenizer.new(text, punctuation: 'semi').tokenize
-# => ["\"", "i", "said", ",", "'", "what're", "you", "?", "crazy", "?", "'", "\"", "said", "sandowsky", "\"", "i", "can't", "afford", "to", "do", "that", "\""]
-
-PragmaticTokenizer::Tokenizer.new(text, expand_contractions: true).tokenize
-# => ['"', 'i', 'said', ',', "'", 'what', 'are', 'you', '?', 'crazy', '?', "'", '"', 'said', 'sandowsky', '.', '"', 'i', 'cannot', 'afford', 'to', 'do', 'that', '.', '"']
-
-PragmaticTokenizer::Tokenizer.new(text, 
-  expand_contractions: true, 
-  remove_stop_words: true, 
-  punctuation: 'none'
-).tokenize
-# => ["crazy", "sandowsky", "afford"]
-
-text = "The price is $5.50 and it works for 5 hours."
-PragmaticTokenizer::Tokenizer.new(text, remove_numbers: true).tokenize
-# => ["the", "price", "is", "and", "it", "works", "for", "hours", "."]
-
-text = "Hello ______ ."
-PragmaticTokenizer::Tokenizer.new(text, clean: true).tokenize
-# => ["hello", "."]
-
-text = "Let's test the minimum length."
-PragmaticTokenizer::Tokenizer.new(text, minimum_length: 6).tokenize
-# => ["minimum", "length"]
-```
+##### `remove_domains`
+  **default** = `'false'`
+- `true`  
+  Removes any token that contains a domain.
+- `false`   
+  Leaves tokens as is.
 
 <hr>
 
-#### `#urls`
-Extract only valid URL tokens
-
-**Example Usage**
-```ruby
-text = "Go to http://www.example.com"
-
-PragmaticTokenizer::Tokenizer.new(text).urls
-# => ["http://www.example.com"]
-```
+##### `clean`
+  **default** = `'false'`
+- `true`  
+  Removes tokens consisting of only hypens, underscores, or periods as well as some special characters (®, ©, ™). Also removes long tokens or tokens with a backslash.
+- `false`   
+  Leaves tokens as is.
 
 <hr>
 
-#### `#domains`
-Extract only valid domain tokens
-
-**Example Usage**
-```ruby
-text = "See the breaking news stories about X on cnn.com/europe and english.alarabiya.net, here’s a screenshot: https://t.co/s83k28f29d31s83"
-
-PragmaticTokenizer::Tokenizer.new(text).urls
-# => ["cnn.com/europe", "english.alarabiya.net"]
-```
+##### `hashtags`
+  **default** = `'keep_original'`
+- `:keep_original`  
+  Does not alter the token at all.
+- `:keep_and_clean`   
+  Removes the hashtag (#) prefix from the token.
+- `:remove`   
+  Removes the token completely.
 
 <hr>
 
-#### `#emails`
-Extract only valid email tokens
-
-**Example Usage**
-```ruby
-text = "Please email example@example.com for more info."
-
-PragmaticTokenizer::Tokenizer.new(text).emails
-# => ["example@example.com"]
-```
+##### `mentions`
+  **default** = `'keep_original'`
+- `:keep_original`  
+  Does not alter the token at all.
+- `:keep_and_clean`   
+  Removes the mention (@) prefix from the token.
+- `:remove`   
+  Removes the token completely.
 
 <hr>
 
-#### `#hashtags`
-Extract only valid hashtag tokens
-
-**Example Usage**
-```ruby
-text = "Find me all the #fun #hashtags and give me #backallofthem."
-
-PragmaticTokenizer::Tokenizer.new(text).hashtags
-# => ["#fun", "#hashtags", "#backallofthem"]
-```
+##### `classic_filter`
+  **default** = `'false'`
+- `true`  
+  Removes dots from acronyms and 's from the end of tokens.
+- `false`   
+  Leaves tokens as is.
 
 <hr>
 
-#### `#mentions`
-Extract only valid @ mention tokens
-
-**Example Usage**
-```ruby
-text = "Find me all the @awesome mentions."
-
-PragmaticTokenizer::Tokenizer.new(text).hashtags
-# => ["@awesome"]
-```
+##### `downcase`
+  **default** = `'true'`
 
 <hr>
 
-#### `#emoticons`
-Extract only simple emoticon tokens
-
-**Example Usage**
-```ruby
-text = "Hello ;-) :) 😄"
-
-PragmaticTokenizer::Tokenizer.new(text).emoticons
-# => [";-)", ":)""]
-```
+##### `minimum_length`
+  **default** = `0`  
+  The minimum number of characters a token should be.  
 
 <hr>
 
-#### `#emoji`
-Extract only valid† emoji tokens
-
-*†matches all 1012 single-character Unicode Emoji (all except for two-character flags)*
-
-**Example Usage**
-```ruby
-text = "Return the emoji 👿😍😱🐔🌚."
-
-PragmaticTokenizer::Tokenizer.new(text).emoticons
-# => ["👿", "😍", "😱", "🐔", "🌚"]
-```
+##### `long_word_split`
+  **default** = `nil`  
+  The number of characters after which a token should be split at hypens or underscores.
 
 ## Language Support
 

data/lib/pragmatic_tokenizer/ending_punctuation_separator.rb

@@ -0,0 +1,31 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticTokenizer
+  # This class separates ending punctuation from a token
+  class EndingPunctuationSeparator
+    attr_reader :tokens
+    def initialize(tokens:)
+      @tokens = tokens
+    end
+
+    def separate
+      cleaned_tokens = []
+      tokens.each do |a|
+        split_punctuation = a.scan(/(?<=\S)[。．！!?？]+$/)
+        if split_punctuation[0].nil?
+          cleaned_tokens << a
+        else
+          cleaned_tokens << a.tr(split_punctuation[0],'')
+          if split_punctuation[0].length.eql?(1)
+            cleaned_tokens << split_punctuation[0]
+          else
+            split_punctuation[0].split("").each do |s|
+              cleaned_tokens << s
+            end
+          end
+        end
+      end
+      cleaned_tokens
+    end
+  end
+end

data/lib/pragmatic_tokenizer/full_stop_separator.rb

@@ -0,0 +1,38 @@
+# -*- encoding : utf-8 -*-
+
+module PragmaticTokenizer
+  # This class separates true full stops while ignoring
+  # periods that are part of an abbreviation
+  class FullStopSeparator
+    attr_reader :tokens, :abbreviations
+    def initialize(tokens:, abbreviations:)
+      @tokens = tokens
+      @abbreviations = abbreviations
+    end
+
+    def separate
+      abbr = {}
+      abbreviations.each do |i|
+        abbr[i] = true
+      end
+      cleaned_tokens = []
+      tokens.each_with_index do |_t, i|
+        if tokens[i + 1] && tokens[i] =~ /\A(.+)\.\z/
+          w = $1
+          unless abbr[Unicode::downcase(w)] || w =~ /\A[a-z]\z/i ||
+            w =~ /[a-z](?:\.[a-z])+\z/i
+            cleaned_tokens <<  w
+            cleaned_tokens << '.'
+            next
+          end
+        end
+        cleaned_tokens << tokens[i]
+      end
+      if cleaned_tokens[-1] && cleaned_tokens[-1] =~ /\A(.*\w)\.\z/
+        cleaned_tokens[-1] = $1
+        cleaned_tokens.push '.'
+      end
+      cleaned_tokens
+    end
+  end
+end

data/lib/pragmatic_tokenizer/languages/arabic.rb

@@ -2,9 +2,9 @@ module PragmaticTokenizer
   module Languages
     module Arabic
       include Languages::Common
-      ABBREVIATIONS = ['ا', 'ا. د', 'ا.د', 'ا.ش.ا', 'ا.ش.ا', 'إلخ', 'ت.ب', 'ت.ب', 'ج.ب', 'جم', 'ج.ب', 'ج.م.ع', 'ج.م.ع', 'س.ت', 'س.ت', 'سم', 'ص.ب.', 'ص.ب', 'كج.', 'كلم.', 'م', 'م.ب', 'م.ب', 'ه', 'د‪']
-      STOP_WORDS = ["فى", "في", "كل", "لم", "لن", "له", "من", "هو", "هي", "قوة", "كما", "لها", "منذ", "وقد", "ولا", "نفسه", "لقاء", "مقابل", "هناك", "وقال", "وكان", "نهاية", "وقالت", "وكانت", "للامم", "فيه", "كلم", "لكن", "وفي", "وقف", "ولم", "ومن", "وهو", "وهي", "يوم", "فيها", "منها", "مليار", "لوكالة", "يكون", "يمكن", "مليون", "حيث", "اكد", "الا", "اما", "امس", "السابق", "التى", "التي", "اكثر", "ايار", "ايضا", "ثلاثة", "الذاتي", "الاخيرة", "الثاني", "الثانية", "الذى", "الذي", "الان", "امام", "ايام", "خلال", "حوالى", "الذين", "الاول", "الاولى", "بين", "ذلك", "دون", "حول", "حين", "الف", "الى", "انه", "اول", "ضمن", "انها", "جميع", "الماضي", "الوقت", "المقبل", "اليوم", "ـ", "ف", "و", "و6", "قد", "لا", "ما", "مع", "مساء", "هذا", "واحد", "واضاف", "واضافت", "فان", "قبل", "قال", "كان", "لدى", "نحو", "هذه", "وان", "واكد", "كانت", "واوضح", "مايو", "ب", "ا", "أ", "،", "عشر", "عدد", "عدة", "عشرة", "عدم", "عام", "عاما", "عن", "عند", "عندما", "على", "عليه", "عليها", "زيارة", "سنة", "سنوات", "تم", "ضد", "بعد", "بعض", "اعادة", "اعلنت", "بسبب", "حتى", "اذا", "احد", "اثر", "برس", "باسم", "غدا", "شخصا", "صباح", "اطار", "اربعة", "اخرى", "بان", "اجل", "غير", "بشكل", "حاليا", "بن", "به", "ثم", "اف", "ان", "او", "اي", "بها", "صفر", "فى"]
-      CONTRACTIONS = {}
+      ABBREVIATIONS = ['ا', 'ا. د', 'ا.د', 'ا.ش.ا', 'ا.ش.ا', 'إلخ', 'ت.ب', 'ت.ب', 'ج.ب', 'جم', 'ج.ب', 'ج.م.ع', 'ج.م.ع', 'س.ت', 'س.ت', 'سم', 'ص.ب.', 'ص.ب', 'كج.', 'كلم.', 'م', 'م.ب', 'م.ب', 'ه', 'د‪'].freeze
+      STOP_WORDS = ["فى", "في", "كل", "لم", "لن", "له", "من", "هو", "هي", "قوة", "كما", "لها", "منذ", "وقد", "ولا", "نفسه", "لقاء", "مقابل", "هناك", "وقال", "وكان", "نهاية", "وقالت", "وكانت", "للامم", "فيه", "كلم", "لكن", "وفي", "وقف", "ولم", "ومن", "وهو", "وهي", "يوم", "فيها", "منها", "مليار", "لوكالة", "يكون", "يمكن", "مليون", "حيث", "اكد", "الا", "اما", "امس", "السابق", "التى", "التي", "اكثر", "ايار", "ايضا", "ثلاثة", "الذاتي", "الاخيرة", "الثاني", "الثانية", "الذى", "الذي", "الان", "امام", "ايام", "خلال", "حوالى", "الذين", "الاول", "الاولى", "بين", "ذلك", "دون", "حول", "حين", "الف", "الى", "انه", "اول", "ضمن", "انها", "جميع", "الماضي", "الوقت", "المقبل", "اليوم", "ـ", "ف", "و", "و6", "قد", "لا", "ما", "مع", "مساء", "هذا", "واحد", "واضاف", "واضافت", "فان", "قبل", "قال", "كان", "لدى", "نحو", "هذه", "وان", "واكد", "كانت", "واوضح", "مايو", "ب", "ا", "أ", "،", "عشر", "عدد", "عدة", "عشرة", "عدم", "عام", "عاما", "عن", "عند", "عندما", "على", "عليه", "عليها", "زيارة", "سنة", "سنوات", "تم", "ضد", "بعد", "بعض", "اعادة", "اعلنت", "بسبب", "حتى", "اذا", "احد", "اثر", "برس", "باسم", "غدا", "شخصا", "صباح", "اطار", "اربعة", "اخرى", "بان", "اجل", "غير", "بشكل", "حاليا", "بن", "به", "ثم", "اف", "ان", "او", "اي", "بها", "صفر", "فى"].freeze
+      CONTRACTIONS = {}.freeze
     end
   end
 end

data/lib/pragmatic_tokenizer/languages/bulgarian.rb

@@ -2,9 +2,9 @@ module PragmaticTokenizer
   module Languages
     module Bulgarian
       include Languages::Common
-      ABBREVIATIONS = ["акад", "ал", "б.р", "б.ред", "бел.а", "бел.пр", "бр", "бул", "в", "вж", "вкл", "вм", "вр", "г", "ген", "гр", "дж", "дм", "доц", "др", "ем", "заб", "зам", "инж", "к.с", "кв", "кв.м", "кг", "км", "кор", "куб", "куб.м", "л", "лв", "м", "м.г", "мин", "млн", "млрд", "мм", "н.с", "напр", "пл", "полк", "проф", "р", "рис", "с", "св", "сек", "см", "сп", "срв", "ст", "стр", "т", "т.г", "т.е", "т.н", "т.нар", "табл", "тел", "у", "ул", "фиг", "ха", "хил", "ч", "чл", "щ.д"]
-      STOP_WORDS = ["а", "автентичен", "аз", "ако", "ала", "бе", "без", "беше", "би", "бивш", "бивша", "бившо", "бил", "била", "били", "било", "благодаря", "близо", "бъдат", "бъде", "бяха", "в", "вас", "ваш", "ваша", "вероятно", "вече", "взема", "ви", "вие", "винаги", "внимава", "време", "все", "всеки", "всички", "всичко", "всяка", "във", "въпреки", "върху", "г", "г.", "ги", "главен", "главна", "главно", "глас", "го", "година", "години", "годишен", "д", "да", "дали", "два", "двама", "двамата", "две", "двете", "ден", "днес", "дни", "до", "добра", "добре", "добро", "добър", "докато", "докога", "дори", "досега", "доста", "друг", "друга", "други", "е", "евтин", "едва", "един", "една", "еднаква", "еднакви", "еднакъв", "едно", "екип", "ето", "живот", "за", "забавям", "зад", "заедно", "заради", "засега", "заспал", "затова", "защо", "защото", "и", "из", "или", "им", "има", "имат", "иска", "й", "каза", "как", "каква", "какво", "както", "какъв", "като", "кога", "когато", "което", "които", "кой", "който", "колко", "която", "къде", "където", "към", "лесен", "лесно", "ли", "лош", "м", "май", "малко", "ме", "между", "мек", "мен", "месец", "ми", "много", "мнозина", "мога", "могат", "може", "мокър", "моля", "момента", "му", "н", "на", "над", "назад", "най", "направи", "напред", "например", "нас", "не", "него", "нещо", "нея", "ни", "ние", "никой", "нито", "нищо", "но", "нов", "нова", "нови", "новина", "някои", "някой", "няколко", "няма", "обаче", "около", "освен", "особено", "от", "отгоре", "отново", "още", "пак", "по", "повече", "повечето", "под", "поне", "поради", "после", "почти", "прави", "пред", "преди", "през", "при", "пък", "първата", "първи", "първо", "пъти", "равен", "равна", "с", "са", "сам", "само", "се", "сега", "си", "син", "скоро", "след", "следващ", "сме", "смях", "според", "сред", "срещу", "сте", "съм", "със", "също", "т", "т.н.", "тази", "така", "такива", "такъв", "там", "твой", "те", "тези", "ти", "то", "това", "тогава", "този", "той", "толкова", "точно", "три", "трябва", "тук", "тъй", "тя", "тях", "у", "утре", "харесва", "хиляди", "ч", "часа", "че", "често", "чрез", "ще", "щом", "юмрук", "я", "як"]
-      CONTRACTIONS = {}
+      ABBREVIATIONS = ["акад", "ал", "б.р", "б.ред", "бел.а", "бел.пр", "бр", "бул", "в", "вж", "вкл", "вм", "вр", "г", "ген", "гр", "дж", "дм", "доц", "др", "ем", "заб", "зам", "инж", "к.с", "кв", "кв.м", "кг", "км", "кор", "куб", "куб.м", "л", "лв", "м", "м.г", "мин", "млн", "млрд", "мм", "н.с", "напр", "пл", "полк", "проф", "р", "рис", "с", "св", "сек", "см", "сп", "срв", "ст", "стр", "т", "т.г", "т.е", "т.н", "т.нар", "табл", "тел", "у", "ул", "фиг", "ха", "хил", "ч", "чл", "щ.д"].freeze
+      STOP_WORDS = ["а", "автентичен", "аз", "ако", "ала", "бе", "без", "беше", "би", "бивш", "бивша", "бившо", "бил", "била", "били", "било", "благодаря", "близо", "бъдат", "бъде", "бяха", "в", "вас", "ваш", "ваша", "вероятно", "вече", "взема", "ви", "вие", "винаги", "внимава", "време", "все", "всеки", "всички", "всичко", "всяка", "във", "въпреки", "върху", "г", "г.", "ги", "главен", "главна", "главно", "глас", "го", "година", "години", "годишен", "д", "да", "дали", "два", "двама", "двамата", "две", "двете", "ден", "днес", "дни", "до", "добра", "добре", "добро", "добър", "докато", "докога", "дори", "досега", "доста", "друг", "друга", "други", "е", "евтин", "едва", "един", "една", "еднаква", "еднакви", "еднакъв", "едно", "екип", "ето", "живот", "за", "забавям", "зад", "заедно", "заради", "засега", "заспал", "затова", "защо", "защото", "и", "из", "или", "им", "има", "имат", "иска", "й", "каза", "как", "каква", "какво", "както", "какъв", "като", "кога", "когато", "което", "които", "кой", "който", "колко", "която", "къде", "където", "към", "лесен", "лесно", "ли", "лош", "м", "май", "малко", "ме", "между", "мек", "мен", "месец", "ми", "много", "мнозина", "мога", "могат", "може", "мокър", "моля", "момента", "му", "н", "на", "над", "назад", "най", "направи", "напред", "например", "нас", "не", "него", "нещо", "нея", "ни", "ние", "никой", "нито", "нищо", "но", "нов", "нова", "нови", "новина", "някои", "някой", "няколко", "няма", "обаче", "около", "освен", "особено", "от", "отгоре", "отново", "още", "пак", "по", "повече", "повечето", "под", "поне", "поради", "после", "почти", "прави", "пред", "преди", "през", "при", "пък", "първата", "първи", "първо", "пъти", "равен", "равна", "с", "са", "сам", "само", "се", "сега", "си", "син", "скоро", "след", "следващ", "сме", "смях", "според", "сред", "срещу", "сте", "съм", "със", "също", "т", "т.н.", "тази", "така", "такива", "такъв", "там", "твой", "те", "тези", "ти", "то", "това", "тогава", "този", "той", "толкова", "точно", "три", "трябва", "тук", "тъй", "тя", "тях", "у", "утре", "харесва", "хиляди", "ч", "часа", "че", "често", "чрез", "ще", "щом", "юмрук", "я", "як"].freeze
+      CONTRACTIONS = {}.freeze
     end
   end
 end

data/lib/pragmatic_tokenizer/languages/catalan.rb

@@ -2,9 +2,9 @@ module PragmaticTokenizer
   module Languages
     module Catalan
       include Languages::Common
-      ABBREVIATIONS = []
-      STOP_WORDS = ["a", "abans", "algun", "alguna", "algunes", "alguns", "altre", "amb", "ambdós", "anar", "ans", "aquell", "aquelles", "aquells", "aquí", "bastant", "bé", "cada", "com", "consegueixo", "conseguim", "conseguir", "consigueix", "consigueixen", "consigueixes", "dalt", "de", "des de", "dins", "el", "elles", "ells", "els", "en", "ens", "entre", "era", "erem", "eren", "eres", "es", "és", "éssent", "està", "estan", "estat", "estava", "estem", "esteu", "estic", "ets", "fa", "faig", "fan", "fas", "fem", "fer", "feu", "fi", "haver", "i", "inclòs", "jo", "la", "les", "llarg", "llavors", "mentre", "meu", "mode", "molt", "molts", "nosaltres", "o", "on", "per", "per que", "però", "perquè", "podem", "poden", "poder", "podeu", "potser", "primer", "puc", "quan", "quant", "qui", "sabem", "saben", "saber", "sabeu", "sap", "saps", "sense", "ser", "seu", "seus", "si", "soc", "solament", "sols", "som", "sota", "també", "te", "tene", "tenim", "tenir", "teniu", "teu", "tinc", "tot", "últim", "un", "una", "unes", "uns", "ús", "va", "vaig", "van", "vosaltres"]
-      CONTRACTIONS = {}
+      ABBREVIATIONS = [].freeze
+      STOP_WORDS = ["a", "abans", "algun", "alguna", "algunes", "alguns", "altre", "amb", "ambdós", "anar", "ans", "aquell", "aquelles", "aquells", "aquí", "bastant", "bé", "cada", "com", "consegueixo", "conseguim", "conseguir", "consigueix", "consigueixen", "consigueixes", "dalt", "de", "des de", "dins", "el", "elles", "ells", "els", "en", "ens", "entre", "era", "erem", "eren", "eres", "es", "és", "éssent", "està", "estan", "estat", "estava", "estem", "esteu", "estic", "ets", "fa", "faig", "fan", "fas", "fem", "fer", "feu", "fi", "haver", "i", "inclòs", "jo", "la", "les", "llarg", "llavors", "mentre", "meu", "mode", "molt", "molts", "nosaltres", "o", "on", "per", "per que", "però", "perquè", "podem", "poden", "poder", "podeu", "potser", "primer", "puc", "quan", "quant", "qui", "sabem", "saben", "saber", "sabeu", "sap", "saps", "sense", "ser", "seu", "seus", "si", "soc", "solament", "sols", "som", "sota", "també", "te", "tene", "tenim", "tenir", "teniu", "teu", "tinc", "tot", "últim", "un", "una", "unes", "uns", "ús", "va", "vaig", "van", "vosaltres"].freeze
+      CONTRACTIONS = {}.freeze
     end
   end
 end

data/lib/pragmatic_tokenizer/languages/common.rb

@@ -1,17 +1,23 @@
 module PragmaticTokenizer
   module Languages
     module Common
-      PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»', '/', '›', '‹', '^', '”']
-      PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚", "”" => "⚘" }
-      SEMI_PUNCTUATION = ['。', '．', '.']
-      ROMAN_NUMERALS = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix', 'xx', 'xxi', 'xxii', 'xxiii', 'xxiv', 'xxv', 'xxvi', 'xxvii', 'xxviii', 'xxix', 'xxx', 'xxxi', 'xxxii', 'xxxiii', 'xxxiv', 'xxxv', 'xxxvi', 'xxxvii', 'xxxviii', 'xxxix', 'xl', 'xli', 'xlii', 'xliii', 'xliv', 'xlv', 'xlvi', 'xlvii', 'xlviii', 'xlix', 'l', 'li', 'lii', 'liii', 'liv', 'lv', 'lvi', 'lvii', 'lviii', 'lix', 'lx', 'lxi', 'lxii', 'lxiii', 'lxiv', 'lxv', 'lxvi', 'lxvii', 'lxviii', 'lxix', 'lxx', 'lxxi', 'lxxii', 'lxxiii', 'lxxiv', 'lxxv', 'lxxvi', 'lxxvii', 'lxxviii', 'lxxix', 'lxxx', 'lxxxi', 'lxxxii', 'lxxxiii', 'lxxxiv', 'lxxxv', 'lxxxvi', 'lxxxvii', 'lxxxviii', 'lxxxix', 'xc', 'xci', 'xcii', 'xciii', 'xciv', 'xcv', 'xcvi', 'xcvii', 'xcviii', 'xcix']
-      SPECIAL_CHARACTERS = ['®', '©', '™']
-      ABBREVIATIONS = []
-      STOP_WORDS = []
-      CONTRACTIONS = {}
+      PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»', '/', '›', '‹', '^', '”'].freeze
+      PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚", "”" => "⚘" }.freeze
+      SEMI_PUNCTUATION = ['。', '．', '.'].freeze
+      ROMAN_NUMERALS = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix', 'xx', 'xxi', 'xxii', 'xxiii', 'xxiv', 'xxv', 'xxvi', 'xxvii', 'xxviii', 'xxix', 'xxx', 'xxxi', 'xxxii', 'xxxiii', 'xxxiv', 'xxxv', 'xxxvi', 'xxxvii', 'xxxviii', 'xxxix', 'xl', 'xli', 'xlii', 'xliii', 'xliv', 'xlv', 'xlvi', 'xlvii', 'xlviii', 'xlix', 'l', 'li', 'lii', 'liii', 'liv', 'lv', 'lvi', 'lvii', 'lviii', 'lix', 'lx', 'lxi', 'lxii', 'lxiii', 'lxiv', 'lxv', 'lxvi', 'lxvii', 'lxviii', 'lxix', 'lxx', 'lxxi', 'lxxii', 'lxxiii', 'lxxiv', 'lxxv', 'lxxvi', 'lxxvii', 'lxxviii', 'lxxix', 'lxxx', 'lxxxi', 'lxxxii', 'lxxxiii', 'lxxxiv', 'lxxxv', 'lxxxvi', 'lxxxvii', 'lxxxviii', 'lxxxix', 'xc', 'xci', 'xcii', 'xciii', 'xciv', 'xcv', 'xcvi', 'xcvii', 'xcviii', 'xcix'].freeze
+      SPECIAL_CHARACTERS = ['®', '©', '™'].freeze
+      ABBREVIATIONS = [].freeze
+      STOP_WORDS = [].freeze
+      CONTRACTIONS = {}.freeze
+      EMOJI_REGEX = /[\u{203C}\u{2049}\u{20E3}\u{2122}\u{2139}\u{2194}-\u{2199}\u{21A9}-\u{21AA}\u{231A}-\u{231B}\u{23E9}-\u{23EC}\u{23F0}\u{23F3}\u{24C2}\u{25AA}-\u{25AB}\u{25B6}\u{25C0}\u{25FB}-\u{25FE}\u{2600}-\u{2601}\u{260E}\u{2611}\u{2614}-\u{2615}\u{261D}\u{263A}\u{2648}-\u{2653}\u{2660}\u{2663}\u{2665}-\u{2666}\u{2668}\u{267B}\u{267F}\u{2693}\u{26A0}-\u{26A1}\u{26AA}-\u{26AB}\u{26BD}-\u{26BE}\u{26C4}-\u{26C5}\u{26CE}\u{26D4}\u{26EA}\u{26F2}-\u{26F3}\u{26F5}\u{26FA}\u{26FD}\u{2702}\u{2705}\u{2708}-\u{270C}\u{270F}\u{2712}\u{2714}\u{2716}\u{2728}\u{2733}-\u{2734}\u{2744}\u{2747}\u{274C}\u{274E}\u{2753}-\u{2755}\u{2757}\u{2764}\u{2795}-\u{2797}\u{27A1}\u{27B0}\u{2934}-\u{2935}\u{2B05}-\u{2B07}\u{2B1B}-\u{2B1C}\u{2B50}\u{2B55}\u{3030}\u{303D}\u{3297}\u{3299}\u{1F004}\u{1F0CF}\u{1F170}-\u{1F171}\u{1F17E}-\u{1F17F}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E7}-\u{1F1EC}\u{1F1EE}-\u{1F1F0}\u{1F1F3}\u{1F1F5}\u{1F1F7}-\u{1F1FA}\u{1F201}-\u{1F202}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F23A}\u{1F250}-\u{1F251}\u{1F300}-\u{1F320}\u{1F330}-\u{1F335}\u{1F337}-\u{1F37C}\u{1F380}-\u{1F393}\u{1F3A0}-\u{1F3C4}\u{1F3C6}-\u{1F3CA}\u{1F3E0}-\u{1F3F0}\u{1F400}-\u{1F43E}\u{1F440}\u{1F442}-\u{1F4F7}\u{1F4F9}-\u{1F4FC}\u{1F500}-\u{1F507}\u{1F509}-\u{1F53D}\u{1F550}-\u{1F567}\u{1F5FB}-\u{1F640}\u{1F645}-\u{1F64F}\u{1F680}-\u{1F68A}]/
+      PREFIX_EMOJI_REGEX = /(?<=\S)(?=[\u{203C}\u{2049}\u{20E3}\u{2122}\u{2139}\u{2194}-\u{2199}\u{21A9}-\u{21AA}\u{231A}-\u{231B}\u{23E9}-\u{23EC}\u{23F0}\u{23F3}\u{24C2}\u{25AA}-\u{25AB}\u{25B6}\u{25C0}\u{25FB}-\u{25FE}\u{2600}-\u{2601}\u{260E}\u{2611}\u{2614}-\u{2615}\u{261D}\u{263A}\u{2648}-\u{2653}\u{2660}\u{2663}\u{2665}-\u{2666}\u{2668}\u{267B}\u{267F}\u{2693}\u{26A0}-\u{26A1}\u{26AA}-\u{26AB}\u{26BD}-\u{26BE}\u{26C4}-\u{26C5}\u{26CE}\u{26D4}\u{26EA}\u{26F2}-\u{26F3}\u{26F5}\u{26FA}\u{26FD}\u{2702}\u{2705}\u{2708}-\u{270C}\u{270F}\u{2712}\u{2714}\u{2716}\u{2728}\u{2733}-\u{2734}\u{2744}\u{2747}\u{274C}\u{274E}\u{2753}-\u{2755}\u{2757}\u{2764}\u{2795}-\u{2797}\u{27A1}\u{27B0}\u{2934}-\u{2935}\u{2B05}-\u{2B07}\u{2B1B}-\u{2B1C}\u{2B50}\u{2B55}\u{3030}\u{303D}\u{3297}\u{3299}\u{1F004}\u{1F0CF}\u{1F170}-\u{1F171}\u{1F17E}-\u{1F17F}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E7}-\u{1F1EC}\u{1F1EE}-\u{1F1F0}\u{1F1F3}\u{1F1F5}\u{1F1F7}-\u{1F1FA}\u{1F201}-\u{1F202}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F23A}\u{1F250}-\u{1F251}\u{1F300}-\u{1F320}\u{1F330}-\u{1F335}\u{1F337}-\u{1F37C}\u{1F380}-\u{1F393}\u{1F3A0}-\u{1F3C4}\u{1F3C6}-\u{1F3CA}\u{1F3E0}-\u{1F3F0}\u{1F400}-\u{1F43E}\u{1F440}\u{1F442}-\u{1F4F7}\u{1F4F9}-\u{1F4FC}\u{1F500}-\u{1F507}\u{1F509}-\u{1F53D}\u{1F550}-\u{1F567}\u{1F5FB}-\u{1F640}\u{1F645}-\u{1F64F}\u{1F680}-\u{1F68A}])/
+      POSTFIX_EMOJI_REGEX = /(?<=[\u{203C}\u{2049}\u{20E3}\u{2122}\u{2139}\u{2194}-\u{2199}\u{21A9}-\u{21AA}\u{231A}-\u{231B}\u{23E9}-\u{23EC}\u{23F0}\u{23F3}\u{24C2}\u{25AA}-\u{25AB}\u{25B6}\u{25C0}\u{25FB}-\u{25FE}\u{2600}-\u{2601}\u{260E}\u{2611}\u{2614}-\u{2615}\u{261D}\u{263A}\u{2648}-\u{2653}\u{2660}\u{2663}\u{2665}-\u{2666}\u{2668}\u{267B}\u{267F}\u{2693}\u{26A0}-\u{26A1}\u{26AA}-\u{26AB}\u{26BD}-\u{26BE}\u{26C4}-\u{26C5}\u{26CE}\u{26D4}\u{26EA}\u{26F2}-\u{26F3}\u{26F5}\u{26FA}\u{26FD}\u{2702}\u{2705}\u{2708}-\u{270C}\u{270F}\u{2712}\u{2714}\u{2716}\u{2728}\u{2733}-\u{2734}\u{2744}\u{2747}\u{274C}\u{274E}\u{2753}-\u{2755}\u{2757}\u{2764}\u{2795}-\u{2797}\u{27A1}\u{27B0}\u{2934}-\u{2935}\u{2B05}-\u{2B07}\u{2B1B}-\u{2B1C}\u{2B50}\u{2B55}\u{3030}\u{303D}\u{3297}\u{3299}\u{1F004}\u{1F0CF}\u{1F170}-\u{1F171}\u{1F17E}-\u{1F17F}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E7}-\u{1F1EC}\u{1F1EE}-\u{1F1F0}\u{1F1F3}\u{1F1F5}\u{1F1F7}-\u{1F1FA}\u{1F201}-\u{1F202}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F23A}\u{1F250}-\u{1F251}\u{1F300}-\u{1F320}\u{1F330}-\u{1F335}\u{1F337}-\u{1F37C}\u{1F380}-\u{1F393}\u{1F3A0}-\u{1F3C4}\u{1F3C6}-\u{1F3CA}\u{1F3E0}-\u{1F3F0}\u{1F400}-\u{1F43E}\u{1F440}\u{1F442}-\u{1F4F7}\u{1F4F9}-\u{1F4FC}\u{1F500}-\u{1F507}\u{1F509}-\u{1F53D}\u{1F550}-\u{1F567}\u{1F5FB}-\u{1F640}\u{1F645}-\u{1F64F}\u{1F680}-\u{1F68A}])(?=\S)/
+      EMOTICON_REGEX = /(?::|;|=)(?:-)?(?:\)|D|P)/
 
       class SingleQuotes
         def handle_single_quotes(text)
+          # Convert left quotes to special character except for 'Twas or 'twas
+          text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
           text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
           # Separate right single quotes
           text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text

data/lib/pragmatic_tokenizer/languages/czech.rb

@@ -2,9 +2,9 @@ module PragmaticTokenizer
   module Languages
     module Czech
       include Languages::Common
-      ABBREVIATIONS = []
-      STOP_WORDS = ["ačkoli", "ahoj", "ale", "anebo", "ano", "asi", "aspoň", "během", "bez", "beze", "blízko", "bohužel", "brzo", "bude", "budeme", "budeš", "budete", "budou", "budu", "byl", "byla", "byli", "bylo", "byly", "bys", "čau", "chce", "chceme", "chceš", "chcete", "chci", "chtějí", "chtít", "chut'", "chuti", "co", "čtrnáct", "čtyři", "dál", "dále", "daleko", "děkovat", "děkujeme", "děkuji", "den", "deset", "devatenáct", "devět", "do", "dobrý", "docela", "dva", "dvacet", "dvanáct", "dvě", "hodně", "já", "jak", "jde", "je", "jeden", "jedenáct", "jedna", "jedno", "jednou", "jedou", "jeho", "její", "jejich", "jemu", "jen", "jenom", "ještě", "jestli", "jestliže", "jí", "jich", "jím", "jimi", "jinak", "jsem", "jsi", "jsme", "jsou", "jste", "kam", "kde", "kdo", "kdy", "když", "ke", "kolik", "kromě", "která", "které", "kteří", "který", "kvůli", "má", "mají", "málo", "mám", "máme", "máš", "máte", "mé", "mě", "mezi", "mí", "mít", "mně", "mnou", "moc", "mohl", "mohou", "moje", "moji", "možná", "můj", "musí", "může", "my", "na", "nad", "nade", "nám", "námi", "naproti", "nás", "náš", "naše", "naši", "ne", "ně", "nebo", "nebyl", "nebyla", "nebyli", "nebyly", "něco", "nedělá", "nedělají", "nedělám", "neděláme", "neděláš", "neděláte", "nějak", "nejsi", "někde", "někdo", "nemají", "nemáme", "nemáte", "neměl", "němu", "není", "nestačí", "nevadí", "než", "nic", "nich", "ním", "nimi", "nula", "od", "ode", "on", "ona", "oni", "ono", "ony", "osm", "osmnáct", "pak", "patnáct", "pět", "po", "pořád", "potom", "pozdě", "před", "přes", "přese", "pro", "proč", "prosím", "prostě", "proti", "protože", "rovně", "se", "sedm", "sedmnáct", "šest", "šestnáct", "skoro", "smějí", "smí", "snad", "spolu", "sta", "sté", "sto", "ta", "tady", "tak", "takhle", "taky", "tam", "tamhle", "tamhleto", "tamto", "tě", "tebe", "tebou", "ted'", "tedy", "ten", "ti", "tisíc", "tisíce", "to", "tobě", "tohle", "toto", "třeba", "tři", "třináct", "trošku", "tvá", "tvé", "tvoje", "tvůj", "ty", "určitě", "už", "vám", "vámi", "vás", "váš", "vaše", "vaši", "ve", "večer", "vedle", "vlastně", "všechno", "všichni", "vůbec", "vy", "vždy", "za", "zač", "zatímco", "ze", "že", "aby", "aj", "ani", "az", "budem", "budes", "by", "byt", "ci", "clanek", "clanku", "clanky", "coz", "cz", "dalsi", "design", "dnes", "email", "ho", "jako", "jej", "jeji", "jeste", "ji", "jine", "jiz", "jses", "kdyz", "ktera", "ktere", "kteri", "kterou", "ktery", "ma", "mate", "mi", "mit", "muj", "muze", "nam", "napiste", "nas", "nasi", "nejsou", "neni", "nez", "nove", "novy", "pod", "podle", "pokud", "pouze", "prave", "pred", "pres", "pri", "proc", "proto", "protoze", "prvni", "pta", "re", "si", "strana", "sve", "svych", "svym", "svymi", "take", "takze", "tato", "tema", "tento", "teto", "tim", "timto", "tipy", "toho", "tohoto", "tom", "tomto", "tomuto", "tu", "tuto", "tyto", "uz", "vam", "vas", "vase", "vice", "vsak", "zda", "zde", "zpet", "zpravy", "a", "aniž", "až", "být", "což", "či", "článek", "článku", "články", "další", "i", "jenž", "jiné", "již", "jseš", "jšte", "k", "každý", "kteři", "ku", "me", "ná", "napište", "nechť", "ní", "nové", "nový", "o", "práve", "první", "přede", "při", "s", "sice", "své", "svůj", "svých", "svým", "svými", "také", "takže", "te", "těma", "této", "tím", "tímto", "u", "v", "více", "však", "všechen", "z", "zpět", "zprávy"]
-      CONTRACTIONS = {}
+      ABBREVIATIONS = [].freeze
+      STOP_WORDS = ["ačkoli", "ahoj", "ale", "anebo", "ano", "asi", "aspoň", "během", "bez", "beze", "blízko", "bohužel", "brzo", "bude", "budeme", "budeš", "budete", "budou", "budu", "byl", "byla", "byli", "bylo", "byly", "bys", "čau", "chce", "chceme", "chceš", "chcete", "chci", "chtějí", "chtít", "chut'", "chuti", "co", "čtrnáct", "čtyři", "dál", "dále", "daleko", "děkovat", "děkujeme", "děkuji", "den", "deset", "devatenáct", "devět", "do", "dobrý", "docela", "dva", "dvacet", "dvanáct", "dvě", "hodně", "já", "jak", "jde", "je", "jeden", "jedenáct", "jedna", "jedno", "jednou", "jedou", "jeho", "její", "jejich", "jemu", "jen", "jenom", "ještě", "jestli", "jestliže", "jí", "jich", "jím", "jimi", "jinak", "jsem", "jsi", "jsme", "jsou", "jste", "kam", "kde", "kdo", "kdy", "když", "ke", "kolik", "kromě", "která", "které", "kteří", "který", "kvůli", "má", "mají", "málo", "mám", "máme", "máš", "máte", "mé", "mě", "mezi", "mí", "mít", "mně", "mnou", "moc", "mohl", "mohou", "moje", "moji", "možná", "můj", "musí", "může", "my", "na", "nad", "nade", "nám", "námi", "naproti", "nás", "náš", "naše", "naši", "ne", "ně", "nebo", "nebyl", "nebyla", "nebyli", "nebyly", "něco", "nedělá", "nedělají", "nedělám", "neděláme", "neděláš", "neděláte", "nějak", "nejsi", "někde", "někdo", "nemají", "nemáme", "nemáte", "neměl", "němu", "není", "nestačí", "nevadí", "než", "nic", "nich", "ním", "nimi", "nula", "od", "ode", "on", "ona", "oni", "ono", "ony", "osm", "osmnáct", "pak", "patnáct", "pět", "po", "pořád", "potom", "pozdě", "před", "přes", "přese", "pro", "proč", "prosím", "prostě", "proti", "protože", "rovně", "se", "sedm", "sedmnáct", "šest", "šestnáct", "skoro", "smějí", "smí", "snad", "spolu", "sta", "sté", "sto", "ta", "tady", "tak", "takhle", "taky", "tam", "tamhle", "tamhleto", "tamto", "tě", "tebe", "tebou", "ted'", "tedy", "ten", "ti", "tisíc", "tisíce", "to", "tobě", "tohle", "toto", "třeba", "tři", "třináct", "trošku", "tvá", "tvé", "tvoje", "tvůj", "ty", "určitě", "už", "vám", "vámi", "vás", "váš", "vaše", "vaši", "ve", "večer", "vedle", "vlastně", "všechno", "všichni", "vůbec", "vy", "vždy", "za", "zač", "zatímco", "ze", "že", "aby", "aj", "ani", "az", "budem", "budes", "by", "byt", "ci", "clanek", "clanku", "clanky", "coz", "cz", "dalsi", "design", "dnes", "email", "ho", "jako", "jej", "jeji", "jeste", "ji", "jine", "jiz", "jses", "kdyz", "ktera", "ktere", "kteri", "kterou", "ktery", "ma", "mate", "mi", "mit", "muj", "muze", "nam", "napiste", "nas", "nasi", "nejsou", "neni", "nez", "nove", "novy", "pod", "podle", "pokud", "pouze", "prave", "pred", "pres", "pri", "proc", "proto", "protoze", "prvni", "pta", "re", "si", "strana", "sve", "svych", "svym", "svymi", "take", "takze", "tato", "tema", "tento", "teto", "tim", "timto", "tipy", "toho", "tohoto", "tom", "tomto", "tomuto", "tu", "tuto", "tyto", "uz", "vam", "vas", "vase", "vice", "vsak", "zda", "zde", "zpet", "zpravy", "a", "aniž", "až", "být", "což", "či", "článek", "článku", "články", "další", "i", "jenž", "jiné", "již", "jseš", "jšte", "k", "každý", "kteři", "ku", "me", "ná", "napište", "nechť", "ní", "nové", "nový", "o", "práve", "první", "přede", "při", "s", "sice", "své", "svůj", "svých", "svým", "svými", "také", "takže", "te", "těma", "této", "tím", "tímto", "u", "v", "více", "však", "všechen", "z", "zpět", "zprávy"].freeze
+      CONTRACTIONS = {}.freeze
     end
   end
 end