greeb 0.1.0.rc4 → 0.1.0.rc6

data/.travis.yml

@@ -4,3 +4,4 @@ branches:
     - master
 rvm:
   - 1.9.3
+  - rbx-19mode

data/README.md

@@ -27,8 +27,8 @@ Greeb can help you to solve simple text processing problems:
 ```ruby
 pp Greeb::Tokenizer.new('Hello!').tokens
 =begin
-#<SortedSet: {#<struct Greeb::Entity from=0, to=5, type=:letter>,
- #<struct Greeb::Entity from=5, to=6, type=:punct>}>
+[#<struct Greeb::Entity from=0, to=5, type=:letter>,
+ #<struct Greeb::Entity from=5, to=6, type=:punct>]
 =end
 ```
 
@@ -43,7 +43,7 @@ EOF
 
 pp Greeb::Tokenizer.new(text).tokens
 =begin
-#<SortedSet: {#<struct Greeb::Entity from=0, to=5, type=:letter>,
+[#<struct Greeb::Entity from=0, to=5, type=:letter>,
  #<struct Greeb::Entity from=5, to=6, type=:punct>,
  #<struct Greeb::Entity from=6, to=7, type=:separ>,
  #<struct Greeb::Entity from=7, to=8, type=:letter>,
@@ -70,7 +70,7 @@ pp Greeb::Tokenizer.new(text).tokens
  #<struct Greeb::Entity from=59, to=60, type=:separ>,
  #<struct Greeb::Entity from=60, to=63, type=:letter>,
  #<struct Greeb::Entity from=63, to=64, type=:punct>,
- #<struct Greeb::Entity from=64, to=65, type=:break>}>
+ #<struct Greeb::Entity from=64, to=65, type=:break>]
 =end
 ```
 
@@ -82,8 +82,8 @@ text = 'Hello! How are you?'
 tokenizer = Greeb::Tokenizer.new(text)
 pp Greeb::Segmentator.new(tokenizer).sentences
 =begin
-#<SortedSet: {#<struct Greeb::Entity from=0, to=6, type=:sentence>,
- #<struct Greeb::Entity from=7, to=19, type=:sentence>}>
+[#<struct Greeb::Entity from=0, to=6, type=:sentence>,
+ #<struct Greeb::Entity from=7, to=19, type=:sentence>]
 =end
 ```
 
@@ -111,9 +111,9 @@ pp segmentator.extract(*sentences)
 
 ## Tokens
 
-Greeb operates with entities, tuples of `<from, to, type>`, where
-`from` is a beginning of the entity, `to` is an ending of the entity,
-and `type` is a type of the entity.
+Greeb operates with entities, tuples of *(from, to, kind)*, where
+*from* is a beginning of the entity, *to* is an ending of the entity,
+and *kind* is a type of the entity.
 
 There are several entity types: `:letter`, `:float`, `:integer`,
 `:separ`, `:punct` (for punctuation), `:spunct` (for in-sentence
@@ -132,11 +132,6 @@ systematic and awesome.
 
 ## Build Status [<img src="https://secure.travis-ci.org/eveel/greeb.png"/>](http://travis-ci.org/eveel/greeb)
 
-If you're using [Rubinius](http://rubini.us) please note that it has the
-incompatible `StringScanner` implementation. More information can be
-provided under the following link:
-<https://github.com/rubinius/rubinius/issues/1808>.
-
 ## Dependency Status [<img src="https://gemnasium.com/eveel/greeb.png?travis"/>](https://gemnasium.com/eveel/greeb)
 
 ## Copyright

data/greeb.gemspec

@@ -6,12 +6,12 @@ Gem::Specification.new do |s|
   s.name        = 'greeb'
   s.version     = Greeb::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ['Dmitry A. Ustalov']
+  s.authors     = ['Dmitry Ustalov']
   s.email       = ['dmitry@eveel.ru']
   s.homepage    = 'https://github.com/eveel/greeb'
-  s.summary     = 'Greeb is a simple regexp-based tokenizer.'
-  s.description = 'Greeb is a simple yet awesome regexp-based tokenizer, ' \
-                  'written in Ruby.'
+  s.summary     = 'Greeb is a simple Unicode-aware regexp-based tokenizer.'
+  s.description = 'Greeb is a simple yet awesome and Unicode-aware ' \
+                  'regexp-based tokenizer, written in Ruby.'
 
   s.rubyforge_project = 'greeb'
 

data/lib/greeb.rb

@@ -2,9 +2,9 @@
 
 require 'greeb/version'
 
-# Greeb operates with entities, tuples of `<from, to, kind>`, where
-# `from` is a beginning of the entity, `to` is an ending of the entity,
-# and `kind` is a type of the entity.
+# Greeb operates with entities, tuples of *(from, to, kind)*, where
+# *from* is a beginning of the entity, *to* is an ending of the entity,
+# and *kind* is a type of the entity.
 #
 # There are several entity types: `:letter`, `:float`, `:integer`,
 # `:separ` for separators, `:punct` for punctuation characters,

data/lib/greeb/segmentator.rb

@@ -26,7 +26,7 @@ class Greeb::Segmentator
 
   # Sentences memoization method.
   #
-  # @return [Set<Greeb::Entity>] a set of sentences.
+  # @return [Array<Greeb::Entity>] a set of sentences.
   #
   def sentences
     detect_sentences! unless @sentences
@@ -35,7 +35,7 @@ class Greeb::Segmentator
 
   # Subsentences memoization method.
   #
-  # @return [Set<Greeb::Entity>] a set of subsentences.
+  # @return [Array<Greeb::Entity>] a set of subsentences.
   #
   def subsentences
     detect_subsentences! unless @subsentences
@@ -79,7 +79,7 @@ class Greeb::Segmentator
     # @return [nil] nothing.
     #
     def detect_sentences!
-      @sentences = SortedSet.new
+      @sentences = []
 
       rest = tokens.inject(new_sentence) do |sentence, token|
         if !sentence.from and SENTENCE_DOESNT_START.include?(token.type)

data/lib/greeb/tokenizer.rb

@@ -1,13 +1,29 @@
 # encoding: utf-8
 
-require 'set'
-
 # Greeb's tokenization facilities. Use 'em with love.
 #
 class Greeb::Tokenizer
+  # This runtime error appears when {Greeb::Tokenizer} tries to recognize
+  # unknown character.
+  #
+  class UnknownEntity < RuntimeError
+    attr_reader :text, :pos
+
+    # @private
+    def initialize(text, pos)
+      @text, @pos = text, pos
+    end
+
+    # Generate the real error message.
+    #
+    def to_s
+      'Could not recognize character "%s" @ %d' % [text[pos], pos]
+    end
+  end
+
   # English and Russian letters.
   #
-  LETTERS = /[A-Za-zА-Яа-яЁё]+/u
+  LETTERS = /[\p{L}]+/u
 
   # Floating point values.
   #
@@ -17,21 +33,21 @@ class Greeb::Tokenizer
   #
   INTEGERS = /\d+/u
 
-  # In-subsentence seprator (i.e.: "*" or "=").
+  # In-sentence punctuation character (i.e.: "," or "-").
   #
-  SEPARATORS = /[*=_\/\\ ]+/u
+  SENTENCE_PUNCTUATIONS = /(\,|\-|:|;|\p{Ps}|\p{Pi}|\p{Pf}|\p{Pe})+/u
 
   # Punctuation character (i.e.: "." or "!").
   #
-  PUNCTUATIONS = /(\.|\!|\?)+/u
+  PUNCTUATIONS = /[(\.|\!|\?)]+/u
 
-  # In-sentence punctuation character (i.e.: "," or "-").
+  # In-subsentence seprator (i.e.: "*" or "=").
   #
-  SENTENCE_PUNCTUATIONS = /(\,|\[|\]|\(|\)|\-|:|;)+/u
+  SEPARATORS = /[ \p{Sm}\p{Pc}\p{Po}\p{Pd}]+/u
 
   # Line breaks.
   #
-  BREAKS = /\n+/u
+  BREAKS = /(\r\n|\n|\r)+/u
 
   attr_reader :text, :scanner
   protected :scanner
@@ -46,7 +62,7 @@ class Greeb::Tokenizer
 
   # Tokens memoization method.
   #
-  # @return [Set<Greeb::Entity>] a set of tokens.
+  # @return [Array<Greeb::Entity>] a set of tokens.
   #
   def tokens
     tokenize! unless @tokens
@@ -61,7 +77,7 @@ class Greeb::Tokenizer
     #
     def tokenize!
       @scanner = Greeb::StringScanner.new(text)
-      @tokens = SortedSet.new
+      @tokens = []
       while !scanner.eos?
         parse! LETTERS, :letter or
         parse! FLOATS, :float or
@@ -70,7 +86,7 @@ class Greeb::Tokenizer
         split_parse! PUNCTUATIONS, :punct or
         split_parse! SEPARATORS, :separ or
         split_parse! BREAKS, :break or
-        raise @tokens.inspect
+        raise UnknownEntity.new(text, scanner.char_pos)
       end
     ensure
       scanner.terminate
@@ -83,7 +99,7 @@ class Greeb::Tokenizer
     # @param type [Symbol] a symbol that represents the necessary token
     #   type.
     #
-    # @return [Set<Greeb::Entity>] the modified set of extracted tokens.
+    # @return [Array<Greeb::Entity>] the modified set of extracted tokens.
     #
     def parse! pattern, type
       return false unless token = scanner.scan(pattern)
@@ -101,7 +117,7 @@ class Greeb::Tokenizer
     # @param type [Symbol] a symbol that represents the necessary token
     #   type.
     #
-    # @return [Set<Greeb::Entity>] the modified set of extracted tokens.
+    # @return [Array<Greeb::Entity>] the modified set of extracted tokens.
     #
     def split_parse! pattern, type
       return false unless token = scanner.scan(pattern)

data/lib/greeb/version.rb

@@ -5,5 +5,5 @@
 module Greeb
   # Version of Greeb.
   #
-  VERSION = '0.1.0.rc4'
+  VERSION = '0.1.0.rc6'
 end

data/spec/segmentator_spec.rb

@@ -10,12 +10,12 @@ module Greeb
       subject { Segmentator.new(@tokenizer) }
 
       it 'can be initialized either with Tokenizer' do
-        subject.tokens.must_be_kind_of SortedSet
+        subject.tokens.must_be_kind_of Array
       end
 
       it 'can be initialized either with a set of tokens' do
         subject = Segmentator.new(@tokenizer.tokens)
-        subject.tokens.must_be_kind_of SortedSet
+        subject.tokens.must_be_kind_of Array
       end
 
       it 'should has @tokens ivar' do
@@ -30,7 +30,7 @@ module Greeb
 
       it 'should be segmented' do
         subject.must_equal(
-          SortedSet.new([Entity.new(0, 22, :sentence)])
+          [Entity.new(0, 22, :sentence)]
         )
       end
     end
@@ -42,7 +42,7 @@ module Greeb
 
       it 'should be segmented' do
         subject.must_equal(
-          SortedSet.new([Entity.new(0, 21, :sentence)])
+          [Entity.new(0, 21, :sentence)]
         )
       end
     end
@@ -54,7 +54,7 @@ module Greeb
 
       it 'should be segmented' do
         subject.must_equal(
-          SortedSet.new([Entity.new(6, 27, :sentence)])
+          [Entity.new(6, 27, :sentence)]
         )
       end
     end
@@ -66,8 +66,8 @@ module Greeb
 
       it 'should be segmented' do
         subject.must_equal(
-          SortedSet.new([Entity.new(0, 6,  :sentence),
-                         Entity.new(7, 22, :sentence)])
+          [Entity.new(0, 6,  :sentence),
+           Entity.new(7, 22, :sentence)]
         )
       end
     end
@@ -79,7 +79,7 @@ module Greeb
 
       it 'should be segmented' do
         subject.must_equal(
-          SortedSet.new([Entity.new(2, 17, :sentence)])
+          [Entity.new(2, 17, :sentence)]
         )
       end
     end

data/spec/tokenizer_spec.rb

@@ -32,75 +32,75 @@ module Greeb
       end
 
       it 'should has the tokens set' do
-        subject.tokens.must_be_kind_of SortedSet
+        subject.tokens.must_be_kind_of Array
       end
     end
 
     describe 'tokenization facilities' do
       it 'can handle words' do
         Tokenizer.new('hello').tokens.must_equal(
-          SortedSet.new([Entity.new(0, 5, :letter)])
+          [Entity.new(0, 5, :letter)]
         )
       end
 
       it 'can handle floats' do
         Tokenizer.new('14.88').tokens.must_equal(
-          SortedSet.new([Entity.new(0, 5, :float)])
+          [Entity.new(0, 5, :float)]
         )
       end
 
       it 'can handle integers' do
         Tokenizer.new('1337').tokens.must_equal(
-          SortedSet.new([Entity.new(0, 4, :integer)])
+          [Entity.new(0, 4, :integer)]
         )
       end
 
       it 'can handle words and integers' do
         Tokenizer.new('Hello, I am 18').tokens.must_equal(
-          SortedSet.new([Entity.new(0,  5,  :letter),
-                         Entity.new(5,  6,  :spunct),
-                         Entity.new(6,  7,  :separ),
-                         Entity.new(7,  8,  :letter),
-                         Entity.new(8,  9,  :separ),
-                         Entity.new(9,  11, :letter),
-                         Entity.new(11, 12, :separ),
-                         Entity.new(12, 14, :integer)])
+          [Entity.new(0,  5,  :letter),
+           Entity.new(5,  6,  :spunct),
+           Entity.new(6,  7,  :separ),
+           Entity.new(7,  8,  :letter),
+           Entity.new(8,  9,  :separ),
+           Entity.new(9,  11, :letter),
+           Entity.new(11, 12, :separ),
+           Entity.new(12, 14, :integer)]
         )
       end
 
       it 'can handle multi-line paragraphs' do
         Tokenizer.new("Brateeshka..!\n\nPrines!").tokens.must_equal(
-          SortedSet.new([Entity.new(0,  10, :letter),
-                         Entity.new(10, 12, :punct),
-                         Entity.new(12, 13, :punct),
-                         Entity.new(13, 15, :break),
-                         Entity.new(15, 21, :letter),
-                         Entity.new(21, 22, :punct)])
+          [Entity.new(0,  10, :letter),
+           Entity.new(10, 12, :punct),
+           Entity.new(12, 13, :punct),
+           Entity.new(13, 15, :break),
+           Entity.new(15, 21, :letter),
+           Entity.new(21, 22, :punct)]
         )
       end
 
       it 'can handle separated integers' do
         Tokenizer.new('228/359').tokens.must_equal(
-          SortedSet.new([Entity.new(0, 3, :integer),
-                         Entity.new(3, 4, :separ),
-                         Entity.new(4, 7, :integer)])
+          [Entity.new(0, 3, :integer),
+           Entity.new(3, 4, :separ),
+           Entity.new(4, 7, :integer)]
         )
       end
 
       it 'can deal with Russian language' do
         Tokenizer.new('Братишка, я тебе покушать принёс!').tokens.must_equal(
-          SortedSet.new([Entity.new(0,  8,  :letter),
-                         Entity.new(8,  9,  :spunct),
-                         Entity.new(9,  10, :separ),
-                         Entity.new(10, 11, :letter),
-                         Entity.new(11, 12, :separ),
-                         Entity.new(12, 16, :letter),
-                         Entity.new(16, 17, :separ),
-                         Entity.new(17, 25, :letter),
-                         Entity.new(25, 26, :separ),
-                         Entity.new(26, 32, :letter),
-                         Entity.new(32, 33, :punct)])
-          )
+          [Entity.new(0,  8,  :letter),
+           Entity.new(8,  9,  :spunct),
+           Entity.new(9,  10, :separ),
+           Entity.new(10, 11, :letter),
+           Entity.new(11, 12, :separ),
+           Entity.new(12, 16, :letter),
+           Entity.new(16, 17, :separ),
+           Entity.new(17, 25, :letter),
+           Entity.new(25, 26, :separ),
+           Entity.new(26, 32, :letter),
+           Entity.new(32, 33, :punct)]
+        )
       end
     end
   end

metadata

@@ -1,81 +1,82 @@
 --- !ruby/object:Gem::Specification
 name: greeb
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc4
+  version: 0.1.0.rc6
   prerelease: 6
 platform: ruby
 authors:
-- Dmitry A. Ustalov
+- Dmitry Ustalov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-20 00:00:00.000000000 Z
+date: 2012-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
+  prerelease: false
 - !ruby/object:Gem::Dependency
-  name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '2.11'
+    none: false
   type: :development
-  prerelease: false
+  name: minitest
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '2.11'
+    none: false
+  prerelease: false
 - !ruby/object:Gem::Dependency
-  name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
+  name: simplecov
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
+  prerelease: false
 - !ruby/object:Gem::Dependency
-  name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
+  name: yard
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Greeb is a simple yet awesome regexp-based tokenizer, written in Ruby.
+    none: false
+  prerelease: false
+description: Greeb is a simple yet awesome and Unicode-aware regexp-based tokenizer,
+  written in Ruby.
 email:
 - dmitry@eveel.ru
 executables: []
@@ -105,26 +106,26 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
+      hash: 2757695902770698935
       version: '0'
       segments:
       - 0
-      hash: 1130932854600612903
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
+  none: false
 requirements: []
 rubyforge_project: greeb
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Greeb is a simple regexp-based tokenizer.
+summary: Greeb is a simple Unicode-aware regexp-based tokenizer.
 test_files:
 - spec/segmentator_spec.rb
 - spec/spec_helper.rb