pragmatic_tokenizer 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd6598d349d883708c73d3f51431cf4a516bafdb
-  data.tar.gz: ba8ab51aa218c4d5405ec1db2a539e0fffa478ec
+  metadata.gz: c4b348d207073bd0812a58b29d1277951ffc63dc
+  data.tar.gz: b9c21c948e4164678fe87e666901d327a395c550
 SHA512:
-  metadata.gz: 5e3aa17314b8130edf651229048c502e2834517f3f2d793705232248554511937d0157a9ff2dee57191aadeb66ad7dc5303634e9857f6c9fcf51dd9da43072d7
-  data.tar.gz: 73af6e7bd5807dc7637d662abebb3a27dbf0e2194a51e30e1876d7c931dbaaeab6bc6ee0ddb3cf915a75385fc6613b2a6480d8686f0e2949b7461334453dff87
+  metadata.gz: f67084e2d3cfe8a34b431ba5d250b314d86138400cea242fb40bfe85d7d265df2f5e5ca4d54f5968f3cd56b6a9b21edb7d7368e25f5aef8faee5446bf014facf
+  data.tar.gz: a947af809bce16395da59a10e7f51393794885886528faad65ecc16a3b702874938f6fd02f006bc9045b432c633d67462b7ea41736fb7548e76e4e3de46762bb

data/lib/pragmatic_tokenizer/languages/common.rb

@@ -1,7 +1,7 @@
 module PragmaticTokenizer
   module Languages
     module Common
-      PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»']
+      PUNCTUATION = ['。', '．', '.', '！', '!', '?', '？', '、', '¡', '¿', '„', '“', '[', ']', '"', '#', '$', '%', '&', '(', ')', '*', '+', ',', ':', ';', '<', '=', '>', '@', '^', '_', '`', "'", '{', '|', '}', '~', '-', '«', '»', '/', '›', '‹', '^']
       PUNCTUATION_MAP = { "。" => "♳", "．" => "♴", "." => "♵", "！" => "♶", "!" => "♷", "?" => "♸", "？" => "♹", "、" => "♺", "¡" => "⚀", "¿" => "⚁", "„" => "⚂", "“" => "⚃", "[" => "⚄", "]" => "⚅", "\"" => "☇", "#" => "☈", "$" => "☉", "%" => "☊", "&" => "☋", "(" => "☌", ")" => "☍", "*" => "☠", "+" => "☢", "," => "☣", ":" => "☤", ";" => "☥", "<" => "☦", "=" => "☧", ">" => "☀", "@" => "☁", "^" => "☂", "_" => "☃", "`" => "☄", "'" => "☮", "{" => "♔", "|" => "♕", "}" => "♖", "~" => "♗", "-" => "♘", "«" => "♙", "»" => "♚" }
       SEMI_PUNCTUATION = ['。', '．', '.']
       ROMAN_NUMERALS = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix', 'xx', 'xxi', 'xxii', 'xxiii', 'xxiv', 'xxv', 'xxvi', 'xxvii', 'xxviii', 'xxix', 'xxx', 'xxxi', 'xxxii', 'xxxiii', 'xxxiv', 'xxxv', 'xxxvi', 'xxxvii', 'xxxviii', 'xxxix', 'xl', 'xli', 'xlii', 'xliii', 'xliv', 'xlv', 'xlvi', 'xlvii', 'xlviii', 'xlix', 'l', 'li', 'lii', 'liii', 'liv', 'lv', 'lvi', 'lvii', 'lviii', 'lix', 'lx', 'lxi', 'lxii', 'lxiii', 'lxiv', 'lxv', 'lxvi', 'lxvii', 'lxviii', 'lxix', 'lxx', 'lxxi', 'lxxii', 'lxxiii', 'lxxiv', 'lxxv', 'lxxvi', 'lxxvii', 'lxxviii', 'lxxix', 'lxxx', 'lxxxi', 'lxxxii', 'lxxxiii', 'lxxxiv', 'lxxxv', 'lxxxvi', 'lxxxvii', 'lxxxviii', 'lxxxix', 'xc', 'xci', 'xcii', 'xciii', 'xciv', 'xcv', 'xcvi', 'xcvii', 'xcviii', 'xcix']

data/lib/pragmatic_tokenizer/processor.rb

@@ -15,6 +15,9 @@ module PragmaticTokenizer
       shift_colon(text)
       shift_bracket(text)
       shift_semicolon(text)
+      shift_underscore(text)
+      shift_asterisk(text)
+      shift_at_symbol(text)
       convert_dbl_quotes(text)
       convert_sgl_quotes(text)
       tokens = separate_full_stop(text.squeeze(' ').split.map { |t| convert_sym_to_punct(t.downcase) })
@@ -66,6 +69,21 @@ module PragmaticTokenizer
       text.gsub!(/([\(\[\{\}\]\)])/o) { ' ' + $1 + ' ' } || text
     end
 
+    def shift_underscore(text)
+      text.gsub!(/(?<=\s)\_+/, ' \1') || text
+      text.gsub!(/\_+(?=\s)/, ' \1') || text
+      text.gsub!(/(?<=\A)\_+/, '\1 ') || text
+      text.gsub!(/\_+(?=\z)/, ' \1') || text
+    end
+
+    def shift_asterisk(text)
+      text.gsub!(/\*+/, ' \1 ') || text
+    end
+
+    def shift_at_symbol(text)
+      text.gsub!(/(\A|\s)\@/, '\1 ') || text
+    end
+
     def shift_colon(text)
       return text unless text.include?(':') &&
         !(/\A\d+/ == text.partition(':').last[0]) &&
@@ -81,6 +99,8 @@ module PragmaticTokenizer
 
     def shift_ellipse(text)
       text.gsub!(/(\.\.\.+)/o) { ' ' + $1 + ' ' } || text
+      text.gsub!(/(\.\.+)/o) { ' ' + $1 + ' ' } || text
+      text.gsub!(/(…+)/o) { ' ' + $1 + ' ' } || text
     end
 
     def separate_full_stop(tokens)

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -38,7 +38,7 @@ module PragmaticTokenizer
 
     def tokenize
       return [] unless text
-      cleaner(remove_short_tokens(delete_numbers(delete_roman_numerals(find_contractions(delete_stop_words(remove_punctuation(processor.new(language: language_module).process(text: text))))))))
+      cleaner(remove_short_tokens(delete_numbers(delete_roman_numerals(find_contractions(delete_stop_words(remove_punctuation(processor.new(language: language_module).process(text: text)))))))).reject { |t| t.empty? }
     end
 
     private
@@ -65,8 +65,7 @@ module PragmaticTokenizer
 
     def cleaner(tokens)
       return tokens unless clean
-      tokens.delete_if { |t| t =~ /\A_+\z/ ||
-        t =~ /\A-+\z/ ||
+      tokens.delete_if { |t| t =~ /\A-+\z/ ||
         PragmaticTokenizer::Languages::Common::SPECIAL_CHARACTERS.include?(t) ||
         t =~ /\A\.{2,}\z/ || t.include?("\\") ||
         t.length > 50 ||
@@ -81,7 +80,7 @@ module PragmaticTokenizer
       when 'semi'
         tokens - PragmaticTokenizer::Languages::Common::SEMI_PUNCTUATION
       when 'none'
-        tokens - PragmaticTokenizer::Languages::Common::PUNCTUATION
+        tokens.delete_if { |t| t =~ /\A[[:punct:]]+\z/ || t =~ /\A(‹+|\^+|›+|\++)\z/ } - PragmaticTokenizer::Languages::Common::PUNCTUATION
       when 'only'
         only_punctuation(tokens)
       end

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - Kevin S. Dias
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-08 00:00:00.000000000 Z
+date: 2016-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler