greeb 0.1.0.rc6 → 0.1.0.rc7

data/lib/greeb/tokenizer.rb

@@ -1,6 +1,9 @@
 # encoding: utf-8
 
-# Greeb's tokenization facilities. Use 'em with love.
+# Greeb's tokenization facilities. Use 'em with love. 
+#
+# Unicode character categories been obtained from
+# <http://www.fileformat.info/info/unicode/category/index.htm>.
 #
 class Greeb::Tokenizer
   # This runtime error appears when {Greeb::Tokenizer} tries to recognize
@@ -43,12 +46,16 @@ class Greeb::Tokenizer
 
   # In-subsentence seprator (i.e.: "*" or "=").
   #
-  SEPARATORS = /[ \p{Sm}\p{Pc}\p{Po}\p{Pd}]+/u
+  SEPARATORS = /[ \p{Nl}\p{No}\p{Pd}\p{Pc}\p{Po}\p{Sm}\p{So}\p{Sc}\p{Z}]+/u
 
   # Line breaks.
   #
   BREAKS = /(\r\n|\n|\r)+/u
 
+  # Residuals.
+  #
+  RESIDUALS = /[\p{C}\p{M}\p{Sk}]+/u
+
   attr_reader :text, :scanner
   protected :scanner
 
@@ -86,6 +93,7 @@ class Greeb::Tokenizer
         split_parse! PUNCTUATIONS, :punct or
         split_parse! SEPARATORS, :separ or
         split_parse! BREAKS, :break or
+        parse! RESIDUALS, :residual or
         raise UnknownEntity.new(text, scanner.char_pos)
       end
     ensure

data/lib/greeb/version.rb

@@ -5,5 +5,5 @@
 module Greeb
   # Version of Greeb.
   #
-  VERSION = '0.1.0.rc6'
+  VERSION = '0.1.0.rc7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: greeb
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc6
+  version: 0.1.0.rc7
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-09 00:00:00.000000000 Z
+date: 2012-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -109,7 +109,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      hash: 2757695902770698935
+      hash: 2716089438708653231
       version: '0'
       segments:
       - 0