pragmatic_tokenizer 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4834da7c6c1b1d6c614226840bb2fd5ef8b48b6
-  data.tar.gz: 395868d67e973b2a6e9e28b4b9883c95d1746fe6
+  metadata.gz: 8cba2ce060ad1d9ffc74953a9e3a9504b1c8ed13
+  data.tar.gz: 3d96486358f974ce30165199381b27c3e01f7625
 SHA512:
-  metadata.gz: cc69a6f19545c9f5755df5c996e0625f0e65883fea81f01a877d10fce5f5b4eba8931529aecff9afb2ce56f8b993350d9bad15a94a5bb718db4eeafbbe611a29
-  data.tar.gz: f08442f148d59d98d3970e50ccc3bab2d59c1728fb06d9fefe4670ff5b4aca688168c81c30da3a49d1d800d8b398d53e76d9482c120450f2edc90b1b3c174617
+  metadata.gz: 8cc83be7dc5d9db9dd03d8895ea0a0d7bb1856b0f7b82698362dfb97f8e6cd32c3cbaccbb320ce04cb193594c765ae4cbbce5cb24634f3fe0aced37298ce75c5
+  data.tar.gz: f7edb187f8cc2f60aad58d79eac454b556dec53d77aa86a90d55d054d009783be8b4619da22550a79a01ac91540a7b1e3d5ea509e4c482821e32517f976c6208

data/README.md

@@ -85,7 +85,7 @@ options = {
 <hr>
 
 ##### `remove_stop_words`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes all stop words.
 - `false`   
@@ -94,7 +94,7 @@ options = {
 <hr>
 
 ##### `expand_contractions`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Expands contractions (i.e. i'll -> i will).
 - `false`   
@@ -135,7 +135,7 @@ options = {
 <hr>
 
 ##### `remove_emoji`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes any token that contains an emoji.
 - `false`   
@@ -144,7 +144,7 @@ options = {
 <hr>
 
 ##### `remove_urls`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes any token that contains a URL.
 - `false`   
@@ -153,7 +153,7 @@ options = {
 <hr>
 
 ##### `remove_domains`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes any token that contains a domain.
 - `false`   
@@ -162,7 +162,7 @@ options = {
 <hr>
 
 ##### `remove_domains`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes any token that contains a domain.
 - `false`   
@@ -171,7 +171,7 @@ options = {
 <hr>
 
 ##### `clean`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes tokens consisting of only hypens, underscores, or periods as well as some special characters (®, ©, ™). Also removes long tokens or tokens with a backslash.
 - `false`   
@@ -180,7 +180,7 @@ options = {
 <hr>
 
 ##### `hashtags`
-  **default** = `'keep_original'`
+  **default** = `:keep_original`
 - `:keep_original`  
   Does not alter the token at all.
 - `:keep_and_clean`   
@@ -191,7 +191,7 @@ options = {
 <hr>
 
 ##### `mentions`
-  **default** = `'keep_original'`
+  **default** = `:keep_original`
 - `:keep_original`  
   Does not alter the token at all.
 - `:keep_and_clean`   
@@ -202,7 +202,7 @@ options = {
 <hr>
 
 ##### `classic_filter`
-  **default** = `'false'`
+  **default** = `false`
 - `true`  
   Removes dots from acronyms and 's from the end of tokens.
 - `false`   
@@ -211,7 +211,7 @@ options = {
 <hr>
 
 ##### `downcase`
-  **default** = `'true'`
+  **default** = `true`
 
 <hr>
 

data/lib/pragmatic_tokenizer/languages/common.rb

@@ -2,7 +2,7 @@ module PragmaticTokenizer
   module Languages
     module Common
       PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»', '/', '›', '‹', '^', '”'].freeze
-      PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚", "”" => "⚘" }.freeze
+      PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚", "”" => "⚘", "‘" => "⚭" }.freeze
       SEMI_PUNCTUATION = ['。', '．', '.'].freeze
       ROMAN_NUMERALS = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix', 'xx', 'xxi', 'xxii', 'xxiii', 'xxiv', 'xxv', 'xxvi', 'xxvii', 'xxviii', 'xxix', 'xxx', 'xxxi', 'xxxii', 'xxxiii', 'xxxiv', 'xxxv', 'xxxvi', 'xxxvii', 'xxxviii', 'xxxix', 'xl', 'xli', 'xlii', 'xliii', 'xliv', 'xlv', 'xlvi', 'xlvii', 'xlviii', 'xlix', 'l', 'li', 'lii', 'liii', 'liv', 'lv', 'lvi', 'lvii', 'lviii', 'lix', 'lx', 'lxi', 'lxii', 'lxiii', 'lxiv', 'lxv', 'lxvi', 'lxvii', 'lxviii', 'lxix', 'lxx', 'lxxi', 'lxxii', 'lxxiii', 'lxxiv', 'lxxv', 'lxxvi', 'lxxvii', 'lxxviii', 'lxxix', 'lxxx', 'lxxxi', 'lxxxii', 'lxxxiii', 'lxxxiv', 'lxxxv', 'lxxxvi', 'lxxxvii', 'lxxxviii', 'lxxxix', 'xc', 'xci', 'xcii', 'xciii', 'xciv', 'xcv', 'xcvi', 'xcvii', 'xcviii', 'xcix'].freeze
       SPECIAL_CHARACTERS = ['®', '©', '™'].freeze
@@ -18,6 +18,7 @@ module PragmaticTokenizer
         def handle_single_quotes(text)
           # Convert left quotes to special character except for 'Twas or 'twas
           text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/(\W|^)‘(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' } || text
           text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
           # Separate right single quotes
           text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text

data/lib/pragmatic_tokenizer/languages/english.rb

@@ -99,6 +99,7 @@ module PragmaticTokenizer
         def handle_single_quotes(text)
           # Convert left quotes to special character except for 'Twas or 'twas
           text.gsub!(/(\W|^)'(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text
+          text.gsub!(/(\W|^)‘(?=.*\w)(?!twas)(?!Twas)/o) { $1 ? $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' : ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["‘"] + ' ' } || text
           text.gsub!(/(\W|^)'(?=.*\w)/o, ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"]) || text
           # Separate right single quotes
           text.gsub!(/(\w|\D)'(?!')(?=\W|$)/o) { $1 + ' ' + PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP["'"] + ' ' } || text

data/lib/pragmatic_tokenizer/post_processor.rb

@@ -17,6 +17,7 @@ module PragmaticTokenizer
         .flat_map { |t| t.include?("?") && t !~ /(http|https|www)(\.|:)/ && t.length > 1 ? t.gsub(/\?/, '\1 \2').split(' ').flatten : t }
         .flat_map { |t| t.include?("+") ? t.gsub!(/\+/, '\1 \2').split(' ').flatten : t }
         .flat_map { |t| t =~ /\A\.[^\.]/ && t.length > 1 ? t.gsub(/\./, '\1 ').split(' ').flatten : t }
+        .flat_map { |t| t =~ /\A\:\S{2,}/ ? t.gsub(/\:/, ': ').split(' ').flatten : t }
         .flat_map { |t| t.include?(".") &&
           t !~ /(http|https|www)(\.|:)/ &&
           t !~ /\.(com|net|org|edu|gov|mil|int)/ &&
@@ -35,14 +36,13 @@ module PragmaticTokenizer
           abbreviations.include?(Unicode::downcase(t.split(".")[0] == nil ? '' : t.split(".")[0])) ? t.gsub(/\./, '\1. \2').split(' ').flatten : t }
         .flat_map { |t| t =~ PragmaticTokenizer::Languages::Common::PREFIX_EMOJI_REGEX ? t.gsub(PragmaticTokenizer::Languages::Common::PREFIX_EMOJI_REGEX, '\1 \2').split(' ').flatten : t }
         .flat_map { |t| t =~ PragmaticTokenizer::Languages::Common::POSTFIX_EMOJI_REGEX ? t.gsub(PragmaticTokenizer::Languages::Common::POSTFIX_EMOJI_REGEX, '\1 \2').split(' ').flatten : t }
-        .flat_map { |t| t =~ /\A(#|＃)\S+-/ ? t.gsub(/\-/, '\1 \2').split(' ').flatten : t }
       ).separate
     end
 
     private
 
     def convert_sym_to_punct(token)
-      symbol_matches = /[♳ ♴ ♵ ♶ ♷ ♸ ♹ ♺ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☠ ☢ ☣ ☤ ☥ ☦ ☧ ☀ ☁ ☂ ☃ ☄ ☮ ♔ ♕ ♖ ♗ ♘ ♙ ♚ ⚘]/.match(token)
+      symbol_matches = /[♳ ♴ ♵ ♶ ♷ ♸ ♹ ♺ ⚀ ⚁ ⚂ ⚃ ⚄ ⚅ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☠ ☢ ☣ ☤ ☥ ☦ ☧ ☀ ☁ ☂ ☃ ☄ ☮ ♔ ♕ ♖ ♗ ♘ ♙ ♚ ⚘ ⚭]/.match(token)
       symbol_matches.nil? ? token : token.gsub!(symbol_matches[0], PragmaticTokenizer::Languages::Common::PUNCTUATION_MAP.key(symbol_matches[0]))
     end
   end

data/lib/pragmatic_tokenizer/pre_processor.rb

@@ -16,6 +16,7 @@ module PragmaticTokenizer
       shift_bracket(text)
       shift_semicolon(text)
       shift_caret(text)
+      shift_hashtag(text)
       shift_vertical_bar(text)
       convert_dbl_quotes(text)
       convert_sgl_quotes(text)
@@ -29,6 +30,7 @@ module PragmaticTokenizer
     def shift_comma(text)
       # Shift commas off everything but numbers
       text.gsub!(/,(?!\d)/o, ' , ') || text
+      text.gsub!(/(?<=\D),(?=\S+)/, ' , ') || text
     end
 
     def shift_multiple_dash(text)
@@ -78,6 +80,10 @@ module PragmaticTokenizer
       text.gsub!(/\^/, ' ^ ') || text
     end
 
+    def shift_hashtag(text)
+      text.gsub!(/(?<=\S)(#|＃)(?=\S)/, ' \1\2') || text
+    end
+
     def shift_vertical_bar(text)
       text.gsub!(/\|/, ' | ') || text
     end

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -160,11 +160,17 @@ module PragmaticTokenizer
         .flat_map { |t| t =~ /\_+(?=\z)/ ? t.gsub!(/\_+(?=\z)/, ' \1').split(' ').flatten : t }
         .flat_map { |t| t =~ /\*+/ ? t.gsub!(/\*+/, '\1 ').split(' ').flatten : t }
         .map { |t| t.gsub(/[[:cntrl:]]/, '') }
+        .map { |t| t.gsub(/(?<=\A)\:(?=.+)/, '') }
+        .map { |t| t.gsub(/(?<=\A)!+(?=.+)/, '') }
+        .map { |t| t.gsub(/1+(?=\z)/, '') }
+        .map { |t| t.gsub(/!+(?=\z)/, '') }
+        .map { |t| t.gsub(/!+(1*!*)*(?=\z)/, '') }
         .delete_if { |t| t =~ /\A-+\z/ ||
         PragmaticTokenizer::Languages::Common::SPECIAL_CHARACTERS.include?(t) ||
         t =~ /\A\.{2,}\z/ || t.include?("\\") ||
         t.length > 50 ||
-        (t.length > 1 && t =~ /[&*+<=>^|~]/i)
+        (t.length > 1 && t =~ /[&*+<=>^|~]/i) ||
+        (t.length == 1 && t =~ /\:/)
       }
     end
 
@@ -211,7 +217,7 @@ module PragmaticTokenizer
     end
 
     def remove_emails!
-      @tokens.delete_if { |t| t =~ /\S+(＠|@)\S+/ }.map { |t| t.chomp('.') }
+      @tokens.delete_if { |t| t =~ /\S+(＠|@)\S+\.\S+/ }.map { |t| t.chomp('.') }
     end
 
     def mentions!
@@ -228,6 +234,7 @@ module PragmaticTokenizer
       when 'remove'
         @tokens.delete_if { |t| t =~ /\A(#|＃)/ }
       when 'keep_and_clean'
+        @tokens = @tokens.flat_map { |t| t =~ /\A(#|＃)\S+-/ ? t.gsub(/\-/, '\1 \2').split(' ').flatten : t }
         @tokens.map! { |t| t =~ /\A(#|＃)/ ? t.gsub!(/(?<=\A)(#|＃)/, '') : t }
       end
     end
@@ -237,7 +244,7 @@ module PragmaticTokenizer
     end
 
     def remove_domains!
-      @tokens.delete_if { |t| t =~ /(\s+|\A)[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}(:[0-9]{1,5})?(\/.*)?/ix }
+      @tokens.delete_if { |t| t =~ /(\s+|\A)[a-z0-9]{2,}([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,6}(:[0-9]{1,5})?(\/.*)?/ix }
     end
 
     def split_long_words!

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-18 00:00:00.000000000 Z
+date: 2016-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicode