greeb 0.2.2.pre1 → 0.2.2.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bff91e4559c7b7d1f83ef83212455d4b4b282351
-  data.tar.gz: 867bb10aeb676a2608b701dd57c5a68b215eeaa9
+  metadata.gz: 921db15cfa86d9a5e6bf0e80058d349d708d1750
+  data.tar.gz: 994418e21e9e55ecb42b2329b59efce999281df9
 SHA512:
-  metadata.gz: f116b3773ac59a02e17c6cece192cb5d8cc21ae976e2e78778752c602ef87b4e6cfe9c383e4b09150b11e9c4dbc08788b27a34834c15023d315b08d08e922d26
-  data.tar.gz: 2532a596959945be6793265fc062f9c89d7676df2a28a535e1662b8bf4d848ac66f35e3bf299282dc53e3ab72ce9efe76382079ada9476856da14361ce19df53
+  metadata.gz: df570f263f22cfb2682ab39e758c6cb25a03c88d96bdcca15c79173d2dd937e28406a7baf90162882810973564d8dccb9aaf0962bb577c623ef1eceb3c6f56e4
+  data.tar.gz: 425859becdf31d2dc68552ba2f788675098a218063e627ec5f6b46c3a2a003fe685f75f03385289f482166c01e4e02ce5e7172ba7aa0289ae28f4fdc57645f4b

data/.travis.yml

@@ -1,5 +1,4 @@
 language: ruby
-bundler_args: --without development
 rvm:
   - 2.0.0
   - jruby-19mode

data/Gemfile

@@ -5,7 +5,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development do
-  gem 'simplecov'
+  gem 'sdoc'
 end
 
 group :test do

data/README.md

@@ -1,6 +1,7 @@
 # Greeb
 Greeb [grʲip] is a simple yet awesome and Unicode-aware text segmentator
-that is based on regular expressions.
+that is based on regular expressions. API documentation is available at
+<https://dmchk.github.com/greeb>.
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -175,11 +176,11 @@ There are several entity types at the tokenization stage: `:letter`,
 4. Push to the branch (`git push origin my-new-feature`);
 5. Create new Pull Request.
 
-## Build Status [<img src="https://secure.travis-ci.org/ustalov/greeb.png"/>](http://travis-ci.org/ustalov/greeb)
+## Build Status [<img src="https://secure.travis-ci.org/dmchk/greeb.png"/>](http://travis-ci.org/dmchk/greeb)
 
-## Dependency Status [<img src="https://gemnasium.com/ustalov/greeb.png"/>](https://gemnasium.com/ustalov/greeb)
+## Dependency Status [<img src="https://gemnasium.com/dmchk/greeb.png"/>](https://gemnasium.com/dmchk/greeb)
 
-## Code Climate [<img src="https://codeclimate.com/github/ustalov/greeb.png"/>](https://codeclimate.com/github/ustalov/greeb)
+## Code Climate [<img src="https://codeclimate.com/github/dmchk/greeb.png"/>](https://codeclimate.com/github/dmchk/greeb)
 
 ## Copyright
 

data/bin/greeb

@@ -9,5 +9,5 @@ require 'greeb'
 text = STDIN.read.tap(&:chomp!)
 
 Greeb[text].each do |entity|
-  puts text[entity.from...entity.to] unless entity.type == :separ
+  puts text[entity.from...entity.to] unless [:space, :break].include? entity.type
 end

data/lib/greeb/segmentator.rb

@@ -5,9 +5,9 @@
 #
 class Greeb::Segmentator
   # Sentence does not start from the separator charater, line break
-  # character, and punctuation characters.
+  # character, punctuation characters, and spaces.
   #
-  SENTENCE_DOES_NOT_START = [:separ, :break, :punct, :spunct]
+  SENTENCE_AINT_START = [:separ, :break, :punct, :spunct, :space]
 
   attr_reader :tokens
 
@@ -64,7 +64,7 @@ class Greeb::Segmentator
     collection = []
 
     rest = tokens.inject(sample.dup) do |entity, token|
-      next entity if sentence_does_not_start? entity, token
+      next entity if sentence_aint_start? entity, token
       entity.from = token.from unless entity.from
       next entity if entity.to and entity.to > token.to
 
@@ -72,7 +72,7 @@ class Greeb::Segmentator
         entity.to = find_forward(tokens, token).to
         collection << entity
         entity = sample.dup
-      elsif :separ != token.type
+      elsif ![:separ, :space].include? token.type
         entity.to = token.to
       end
 
@@ -95,8 +95,8 @@ class Greeb::Segmentator
   #
   # @return true or false.
   #
-  def sentence_does_not_start?(entity, token)
-    !entity.from and SENTENCE_DOES_NOT_START.include? token.type
+  def sentence_aint_start?(entity, token)
+    !entity.from and SENTENCE_AINT_START.include? token.type
   end
 
   # Find a forwarding token that has another type.

data/lib/greeb/tokenizer.rb

@@ -31,7 +31,11 @@ module Greeb::Tokenizer
 
   # In-subsentence seprator (i.e.: "*" or "=").
   #
-  SEPARATORS = /[ \p{Nl}\p{No}\p{Pd}\p{Pc}\p{Po}\p{Sm}\p{So}\p{Sc}\p{Z}]+/u
+  SEPARATORS = /[\p{Nl}\p{No}\p{Pd}\p{Pc}\p{Po}\p{Sm}\p{So}\p{Sc}\p{Zl}\p{Zp}]+/u
+
+  # Spaces (i.e.: " " or &nbsp).
+  #
+  SPACES = /[\p{Zs}]+/u
 
   # Line breaks.
   #
@@ -68,7 +72,7 @@ module Greeb::Tokenizer
   # @return [Array<String>] splitted characters.
   #
   def split(token)
-    token.scan(/((.|\n)\2*)/).map(&:first)
+    token.scan(/((.|\n)\2*)/).map!(&:first)
   end
 
   protected
@@ -86,6 +90,7 @@ module Greeb::Tokenizer
     split_parse! scanner, tokens, SENTENCE_PUNCTUATIONS, :spunct or
     split_parse! scanner, tokens, PUNCTUATIONS, :punct or
     split_parse! scanner, tokens, SEPARATORS, :separ or
+    split_parse! scanner, tokens, SPACES, :space or
     split_parse! scanner, tokens, BREAKS, :break or
     parse! scanner, tokens, RESIDUALS, :residual
   end

data/lib/greeb/version.rb

@@ -5,5 +5,5 @@
 module Greeb
   # Version of Greeb.
   #
-  VERSION = '0.2.2.pre1'
+  VERSION = '0.2.2.rc1'
 end

data/spec/core_spec.rb

@@ -11,7 +11,7 @@ module Greeb
     it 'should tokenize text when input is given' do
       Greeb['Hello guys!'].must_equal(
         [Entity.new(0, 5, :letter),
-         Entity.new(5, 6, :separ),
+         Entity.new(5, 6, :space),
          Entity.new(6, 10, :letter),
          Entity.new(10, 11, :punct)]
       )
@@ -20,9 +20,9 @@ module Greeb
     it 'should extract URLs' do
       Greeb['Hello http://nlpub.ru guys!'].must_equal(
         [Entity.new(0, 5, :letter),
-         Entity.new(5, 6, :separ),
+         Entity.new(5, 6, :space),
          Entity.new(6, 21, :url),
-         Entity.new(21, 22, :separ),
+         Entity.new(21, 22, :space),
          Entity.new(22, 26, :letter),
          Entity.new(26, 27, :punct)]
       )
@@ -31,9 +31,9 @@ module Greeb
     it 'should extract e-mails' do
       Greeb['Hello example@example.com guys!'].must_equal(
         [Entity.new(0, 5, :letter),
-         Entity.new(5, 6, :separ),
+         Entity.new(5, 6, :space),
          Entity.new(6, 25, :email),
-         Entity.new(25, 26, :separ),
+         Entity.new(25, 26, :space),
          Entity.new(26, 30, :letter),
          Entity.new(30, 31, :punct)]
       )

data/spec/segmentator_spec.rb

@@ -84,11 +84,11 @@ module Greeb
           ],
           Entity.new(7, 22, :sentence) => [
             Entity.new(7,  8,  :letter),
-            Entity.new(8,  9,  :separ),
+            Entity.new(8,  9,  :space),
             Entity.new(9,  11, :letter),
-            Entity.new(11, 12, :separ),
+            Entity.new(11, 12, :space),
             Entity.new(12, 14, :letter),
-            Entity.new(14, 15, :separ),
+            Entity.new(14, 15, :space),
             Entity.new(15, 21, :letter),
             Entity.new(21, 22, :punct)
           ]

data/spec/spec_helper.rb

@@ -6,13 +6,6 @@ gem 'minitest'
 require 'minitest/autorun'
 require 'minitest/hell'
 
-unless 'true' == ENV['TRAVIS']
-  require 'simplecov'
-  SimpleCov.start do
-    add_filter '/spec/'
-  end
-end
-
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'greeb'
 

data/spec/tokenizer_spec.rb

@@ -35,11 +35,11 @@ module Greeb
         Tokenizer.tokenize('Hello, I am 18').must_equal(
           [Entity.new(0,  5,  :letter),
            Entity.new(5,  6,  :spunct),
-           Entity.new(6,  7,  :separ),
+           Entity.new(6,  7,  :space),
            Entity.new(7,  8,  :letter),
-           Entity.new(8,  9,  :separ),
+           Entity.new(8,  9,  :space),
            Entity.new(9,  11, :letter),
-           Entity.new(11, 12, :separ),
+           Entity.new(11, 12, :space),
            Entity.new(12, 14, :integer)]
         )
       end
@@ -67,13 +67,13 @@ module Greeb
         Tokenizer.tokenize('Братишка, я тебе покушать принёс!').must_equal(
           [Entity.new(0,  8,  :letter),
            Entity.new(8,  9,  :spunct),
-           Entity.new(9,  10, :separ),
+           Entity.new(9,  10, :space),
            Entity.new(10, 11, :letter),
-           Entity.new(11, 12, :separ),
+           Entity.new(11, 12, :space),
            Entity.new(12, 16, :letter),
-           Entity.new(16, 17, :separ),
+           Entity.new(16, 17, :space),
            Entity.new(17, 25, :letter),
-           Entity.new(25, 26, :separ),
+           Entity.new(25, 26, :space),
            Entity.new(26, 32, :letter),
            Entity.new(32, 33, :punct)]
         )

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greeb
 version: !ruby/object:Gem::Version
-  version: 0.2.2.pre1
+  version: 0.2.2.rc1
 platform: ruby
 authors:
 - Dmitry Ustalov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-23 00:00:00.000000000 Z
+date: 2013-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -34,9 +34,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- .rubocop.yml
 - .travis.yml
-- .yardopts
 - Gemfile
 - LICENSE
 - README.md

data/.rubocop.yml

@@ -1,3 +0,0 @@
-# Don't use sprintf instead of %
-FavorSprintf:
-  Enabled: false

data/.yardopts

@@ -1,6 +0,0 @@
---protected
---no-private
--m markdown
--
-README.md
-LICENSE