greeb 0.2.0.pre2 → 0.2.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80ad5b1112ea9576ba14eb52ce56a844e7ee1e94
-  data.tar.gz: 00cd04458ffaf7962d5ed828611cd1decc323852
+  metadata.gz: 618591e00b61f1df11f98bdd045bd650d34ba863
+  data.tar.gz: 88d1b8448e98c18e6d9759e4d992d2fbea7c1d63
 SHA512:
-  metadata.gz: 965fc9a4d9ebbe6b2ed7601fa9aa7f34ccb82f722a48c8bf595786370ff1428c4cfcb4985fc5c9b9f83d5cee1a284e6ca6a440c772e071f0b3245f02717d01e8
-  data.tar.gz: 9a912f214f5c9b12eebbe7a834e3ce9c1e3d3735b1b5fcd9453029301061683019b6f02a2f8da6be70eac26b044a675d6fc87b23b44d3668efc454e7725cd2b0
+  metadata.gz: e8113e47988e80aabfc07314268a5f8220cce88edbf06bd69b35602623c0a310c3c460e300143943596decae621ee69b4909371b9f43a7d9225bceb336bf21f6
+  data.tar.gz: 7ebe3c3e0a603bf1fc0072376c3b2b544b43ae38e31e8bc5ff9e34fcaf362b8c474ba565db67363c09f10a2f4960fdb0bf7a165ee6c0b90d657b3914231cc07a

data/.rubocop.yml

@@ -0,0 +1,3 @@
+# Don't use sprintf instead of %
+FavorSprintf:
+  Enabled: false

data/.travis.yml

@@ -4,4 +4,3 @@ branches:
 rvm:
   - 2.0.0
   - jruby-19mode
-  - rbx-19mode

data/Gemfile

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
-source 'http://rubygems.org'
+source 'https://rubygems.org'
 
 gemspec
+
+group :test do
+  gem 'simplecov'
+end

data/README.md

@@ -1,11 +1,8 @@
-Greeb
-=====
-
-Greeb is a simple yet awesome and Unicode-aware text segmentator
+# Greeb
+Greeb [grʲip] is a simple yet awesome and Unicode-aware text segmentator
 that is based on regular expressions.
 
 ## Installation
-
 Add this line to your application's Gemfile:
 
 ```ruby
@@ -21,8 +18,26 @@ Or install it yourself as:
     $ gem install greeb
 
 ## Usage
+Greeb can help you solve simple text processing problems such as
+tokenization and segmentation.
+
+It is available as a command line application that reads the input
+text from STDIN and prints one token per line into STDOUT.
+
+```
+% echo 'Hello http://nlpub.ru guys, how are you?' | greeb
+Hello
+http://nlpub.ru
+guys
+,
+how
+are
+you
+?
+```
 
-Greeb can help you to solve simple text processing problems:
+### Tokenization API
+Greeb has a very convinient API that makes you happy.
 
 ```ruby
 pp Greeb::Tokenizer.tokenize('Hello!')
@@ -32,7 +47,7 @@ pp Greeb::Tokenizer.tokenize('Hello!')
 =end
 ```
 
-It should be noted that it is possible to process much complex texts:
+It should be noted that it is possible to process much complex texts.
 
 ```ruby
 text =<<-EOF
@@ -74,8 +89,9 @@ pp Greeb::Tokenizer.tokenize(text)
 =end
 ```
 
+### Segmentation API
 Also it can be used to solve the text segmentation problems
-such as sentence detection tasks:
+such as sentence detection tasks.
 
 ```ruby
 text = 'Hello! How are you?'
@@ -88,7 +104,7 @@ pp Greeb::Segmentator.new(tokens).sentences
 ```
 
 It is possible to extract tokens that were processed by the text
-segmentator:
+segmentator.
 
 ```ruby
 text = 'Hello! How are you?'
@@ -109,18 +125,36 @@ pp segmentator.extract(segmentator.sentences)
 =end
 ```
 
-## Tokens
+### Parsing API
+Texts are often include some special entities such as URLs and e-mail
+addresses. Greeb can help you in these strings retrieval.
+
+```ruby
+text = 'My website is http://nlpub.ru and e-mail is example@example.com.'
+
+pp Greeb::Parser.urls(text).map { |e| [e, text[e.from...e.to]] }
+=begin
+[[#<struct Greeb::Entity from=14, to=29, type=:url>, "http://nlpub.ru"]]
+=end
+
+pp Greeb::Parser.emails(text).map { |e| [e, text[e.from...e.to]] }
+=begin
+[[#<struct Greeb::Entity from=44, to=63, type=:email>, "example@example.com"]]
+=end
+```
+
+Please don't use Greeb in spam lists development purposes.
 
+## Tokens
 Greeb operates with entities, tuples of *(from, to, kind)*, where
 *from* is a beginning of the entity, *to* is an ending of the entity,
 and *kind* is a type of the entity.
 
-There are several entity types: `:letter`, `:float`, `:integer`,
-`:separ`, `:punct` (for punctuation), `:spunct` (for in-sentence
-punctuation), and `:break`.
+There are several entity types at the tokenization stage: `:letter`,
+`:float`, `:integer`, `:separ`, `:punct` (for punctuation), `:spunct`
+(for in-sentence punctuation), and `:break`.
 
 ## Contributing
-
 1. Fork it;
 2. Create your feature branch (`git checkout -b my-new-feature`);
 3. Commit your changes (`git commit -am 'Added some feature'`);
@@ -131,6 +165,8 @@ punctuation), and `:break`.
 
 ## Dependency Status [<img src="https://gemnasium.com/ustalov/greeb.png"/>](https://gemnasium.com/ustalov/greeb)
 
+## Code Climate [<img src="https://codeclimate.com/github/ustalov/greeb.png"/>](https://codeclimate.com/github/ustalov/greeb)
+
 ## Copyright
 
 Copyright (c) 2010-2013 [Dmitry Ustalov]. See LICENSE for details.

data/bin/greeb

@@ -1,7 +1,9 @@
 #!/usr/bin/env ruby
 
-$:.unshift File.expand_path('../../lib', __FILE__)
-require 'rubygems'
+if File.exists? File.expand_path('../../.git', __FILE__)
+  $:.unshift File.expand_path('../../lib', __FILE__)
+end
+
 require 'greeb'
 
 text = STDIN.read

data/greeb.gemspec

@@ -17,8 +17,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'minitest', '>= 2.11'
-  s.add_development_dependency 'simplecov'
-  s.add_development_dependency 'yard'
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/greeb/parser.rb

@@ -7,11 +7,11 @@
 module Greeb::Parser
   extend self
 
-  # URL pattern. Not so precise, but IDN-compatible.
-  URL = /\b(([\w-]+:\/\/?|www[.])[^\s()<>]+(?:\([\p{L}\w\d]+\)|([^.\s]|\/)))/ui
+  # An URL pattern. Not so precise, but IDN-compatible.
+  URL = %r{\b(([\w-]+://?|www[.])[^\s()<>]+(?:\([\p{L}\w\d]+\)|([^.\s]|/)))}i
 
-  # Horrible e-mail pattern. 
-  EMAIL = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/ui
+  # A horrible e-mail pattern.
+  EMAIL = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}/i
 
   # Recognize URLs in the input text. Actually, URL is obsolete standard
   # and this code should be rewritten to use the URI concept.

data/lib/greeb/segmentator.rb

@@ -15,7 +15,7 @@ class Greeb::Segmentator
   #
   # @param tokens [Array<Greeb::Entity>] tokens from [Greeb::Tokenizer].
   #
-  def initialize tokens
+  def initialize(tokens)
     @tokens = tokens
   end
 
@@ -44,7 +44,7 @@ class Greeb::Segmentator
   # @return [Hash<Greeb::Entity, Array<Greeb::Entity>>] a hash with
   #   sentences as keys and tokens arrays as values.
   #
-  def extract sentences
+  def extract(sentences)
     Hash[
       sentences.map do |s|
         [s, tokens.select { |t| t.from >= s.from and t.to <= s.to }]
@@ -59,7 +59,7 @@ class Greeb::Segmentator
   # @return [Hash<Greeb::Entity, Array<Greeb::Entity>>] a hash with
   #   sentences as keys and subsentences arrays as values.
   #
-  def subextract sentences
+  def subextract(sentences)
     Hash[
       sentences.map do |s|
         [s, subsentences.select { |ss| ss.from >= s.from and ss.to <= s.to }]
@@ -88,8 +88,7 @@ class Greeb::Segmentator
       if :punct == token.type
         sentence.to = tokens.
           select { |t| t.from >= token.from }.
-          inject(token) { |r, t| break r if t.type != token.type; t }.
-          to
+          inject(token) { |r, t| break r if t.type != token.type; t }.to
 
         @sentences << sentence
         sentence = new_sentence
@@ -100,7 +99,7 @@ class Greeb::Segmentator
       sentence
     end
 
-    nil.tap { @sentences << rest if rest.from and rest.to }
+    nil.tap { @sentences << rest if rest.from && rest.to }
   end
 
   # Implementation of the subsentence detection method. This method
@@ -112,19 +111,18 @@ class Greeb::Segmentator
     @subsentences = SortedSet.new
 
     rest = tokens.inject(new_subsentence) do |subsentence, token|
-      if !subsentence.from and SENTENCE_DOESNT_START.include?(token.type)
+      if !subsentence.from && SENTENCE_DOESNT_START.include?(token.type)
         next subsentence
       end
 
       subsentence.from = token.from unless subsentence.from
 
-      next subsentence if subsentence.to and subsentence.to > token.to
+      next subsentence if subsentence.to && subsentence.to > token.to
 
       if [:punct, :spunct].include? token.type
         subsentence.to = tokens.
           select { |t| t.from >= token.from }.
-          inject(token) { |r, t| break r if t.type != token.type; t }.
-          to
+          inject(token) { |r, t| break r if t.type != token.type; t }.to
 
         @subsentences << subsentence
         subsentence = new_subsentence
@@ -135,7 +133,7 @@ class Greeb::Segmentator
       subsentence
     end
 
-    nil.tap { @subsentences << rest if rest.from and rest.to }
+    nil.tap { @subsentences << rest if rest.from && rest.to }
   end
 
   private

data/lib/greeb/version.rb

@@ -5,5 +5,5 @@
 module Greeb
   # Version of Greeb.
   #
-  VERSION = '0.2.0.pre2'
+  VERSION = '0.2.0.pre3'
 end

data/spec/bin_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe 'CLI' do
+  it 'should do nothing when ran without input' do
+    invoke('').must_be_empty
+  end
+
+  it 'should tokenize text when input is given' do
+    invoke(stdin: 'Hello guys!').must_equal(
+      %w(Hello guys !))
+  end
+
+  it 'should extract URLs' do
+    invoke(stdin: 'Hello http://nlpub.ru guys!').must_equal(
+      %w(Hello http://nlpub.ru guys !))
+  end
+
+  it 'should extract e-mails' do
+    invoke(stdin: 'Hello example@example.com guys!').must_equal(
+      %w(Hello example@example.com guys !))
+  end
+end

data/spec/segmentator_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require File.expand_path('../spec_helper', __FILE__)
+require_relative 'spec_helper'
 
 module Greeb
   describe Segmentator do

data/spec/spec_helper.rb

@@ -2,13 +2,9 @@
 
 require 'rubygems'
 
-$:.unshift File.expand_path('../../lib', __FILE__)
-
-if RUBY_VERSION == '1.8'
-  gem 'minitest'
-end
-
+gem 'minitest'
 require 'minitest/autorun'
+require 'minitest/hell'
 
 unless 'true' == ENV['TRAVIS']
   require 'simplecov'
@@ -17,4 +13,7 @@ unless 'true' == ENV['TRAVIS']
   end
 end
 
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'greeb'
+
+Dir[File.expand_path('../support/**/*.rb', __FILE__)].each { |f| require f }

data/spec/support/invoker.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+
+require 'open3'
+
+# http://dota2.ru/guides/880-invokirkhakha-sanstrajk-ni-azhydal-da/
+#
+class MiniTest::Unit::TestCase
+  # Quas Wex Exort.
+  #
+  def invoke_cache
+    @invoke_cache ||= {}
+  end
+
+  # So begins a new age of knowledge.
+  #
+  def invoke(*argv)
+    return invoke_cache[argv] if invoke_cache.has_key? argv
+
+    arguments = argv.dup
+    options = (arguments.last.is_a? Hash) ? arguments.pop : {}
+    executable = File.expand_path('../../../bin/greeb', __FILE__)
+
+    Open3.popen3(executable, *arguments) do |i, o, *_|
+      i.puts options[:stdin] if options[:stdin]
+      i.close
+      invoke_cache[argv] = o.readlines.map(&:chomp!)
+    end
+  end
+end

data/spec/tokenizer_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-require File.expand_path('../spec_helper', __FILE__)
+require_relative 'spec_helper'
 
 module Greeb
   describe Tokenizer do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greeb
 version: !ruby/object:Gem::Version
-  version: 0.2.0.pre2
+  version: 0.2.0.pre3
 platform: ruby
 authors:
 - Dmitry Ustalov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-21 00:00:00.000000000 Z
+date: 2013-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -38,34 +38,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '2.11'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Greeb is a simple yet awesome and Unicode-aware regexp-based tokenizer,
   written in Ruby.
 email:
@@ -76,6 +48,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rubocop.yml
 - .travis.yml
 - .yardopts
 - Gemfile
@@ -90,9 +63,11 @@ files:
 - lib/greeb/strscan.rb
 - lib/greeb/tokenizer.rb
 - lib/greeb/version.rb
+- spec/bin_spec.rb
 - spec/parser_spec.rb
 - spec/segmentator_spec.rb
 - spec/spec_helper.rb
+- spec/support/invoker.rb
 - spec/tokenizer_spec.rb
 homepage: https://github.com/ustalov/greeb
 licenses: []
@@ -113,13 +88,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: greeb
-rubygems_version: 2.0.3
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: Greeb is a simple Unicode-aware regexp-based tokenizer.
 test_files:
+- spec/bin_spec.rb
 - spec/parser_spec.rb
 - spec/segmentator_spec.rb
 - spec/spec_helper.rb
+- spec/support/invoker.rb
 - spec/tokenizer_spec.rb
 has_rdoc: