greeb 0.2.2.rc1 → 0.2.2.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 921db15cfa86d9a5e6bf0e80058d349d708d1750
-  data.tar.gz: 994418e21e9e55ecb42b2329b59efce999281df9
+  metadata.gz: 043ee2da87958a027caf792058ae1e3e44cc9684
+  data.tar.gz: 8f4a99b26f706badb15fd9e9d5533dd162e090e8
 SHA512:
-  metadata.gz: df570f263f22cfb2682ab39e758c6cb25a03c88d96bdcca15c79173d2dd937e28406a7baf90162882810973564d8dccb9aaf0962bb577c623ef1eceb3c6f56e4
-  data.tar.gz: 425859becdf31d2dc68552ba2f788675098a218063e627ec5f6b46c3a2a003fe685f75f03385289f482166c01e4e02ce5e7172ba7aa0289ae28f4fdc57645f4b
+  metadata.gz: e950167615138975bc9873a729f2486eb506692fcdaefdd3aa828590d261da0d336e04e481c652036df892da305269a047ab076622c315bd17b3c015990dcba7
+  data.tar.gz: 7642fa3892694606792db842b0ea22a8ba13800b71a3e72eaff93d41ab0548f90a6881298aa290a61110941de350ef9941871e7497fda26b4d1e316e3688c50b

data/README.md

@@ -1,7 +1,7 @@
 # Greeb
 Greeb [grʲip] is a simple yet awesome and Unicode-aware text segmentator
 that is based on regular expressions. API documentation is available at
-<https://dmchk.github.com/greeb>.
+<http://rubydoc.info/github/dmchk/greeb/master/frames>.
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -43,8 +43,8 @@ Greeb has a very convinient API that makes you happy.
 ```ruby
 pp Greeb::Tokenizer.tokenize('Hello!')
 =begin
-[#<struct Greeb::Entity from=0, to=5, type=:letter>,
- #<struct Greeb::Entity from=5, to=6, type=:punct>]
+[#<struct Greeb::Span from=0, to=5, type=:letter>,
+ #<struct Greeb::Span from=5, to=6, type=:punct>]
 =end
 ```
 
@@ -59,34 +59,34 @@ EOF
 
 pp Greeb::Tokenizer.tokenize(text)
 =begin
-[#<struct Greeb::Entity from=0, to=5, type=:letter>,
- #<struct Greeb::Entity from=5, to=6, type=:punct>,
- #<struct Greeb::Entity from=6, to=7, type=:separ>,
- #<struct Greeb::Entity from=7, to=8, type=:letter>,
- #<struct Greeb::Entity from=8, to=9, type=:separ>,
- #<struct Greeb::Entity from=9, to=11, type=:letter>,
- #<struct Greeb::Entity from=11, to=12, type=:separ>,
- #<struct Greeb::Entity from=12, to=14, type=:integer>,
- #<struct Greeb::Entity from=14, to=15, type=:punct>,
- #<struct Greeb::Entity from=15, to=16, type=:separ>,
- #<struct Greeb::Entity from=16, to=18, type=:letter>,
- #<struct Greeb::Entity from=18, to=19, type=:separ>,
- #<struct Greeb::Entity from=19, to=28, type=:letter>,
- #<struct Greeb::Entity from=28, to=29, type=:separ>,
- #<struct Greeb::Entity from=29, to=35, type=:letter>,
- #<struct Greeb::Entity from=35, to=36, type=:separ>,
- #<struct Greeb::Entity from=36, to=38, type=:letter>,
- #<struct Greeb::Entity from=38, to=39, type=:separ>,
- #<struct Greeb::Entity from=39, to=44, type=:float>,
- #<struct Greeb::Entity from=44, to=47, type=:punct>,
- #<struct Greeb::Entity from=47, to=49, type=:break>,
- #<struct Greeb::Entity from=49, to=53, type=:letter>,
- #<struct Greeb::Entity from=53, to=54, type=:separ>,
- #<struct Greeb::Entity from=54, to=59, type=:letter>,
- #<struct Greeb::Entity from=59, to=60, type=:separ>,
- #<struct Greeb::Entity from=60, to=63, type=:letter>,
- #<struct Greeb::Entity from=63, to=64, type=:punct>,
- #<struct Greeb::Entity from=64, to=65, type=:break>]
+[#<struct Greeb::Span from=0, to=5, type=:letter>,
+ #<struct Greeb::Span from=5, to=6, type=:punct>,
+ #<struct Greeb::Span from=6, to=7, type=:space>,
+ #<struct Greeb::Span from=7, to=8, type=:letter>,
+ #<struct Greeb::Span from=8, to=9, type=:space>,
+ #<struct Greeb::Span from=9, to=11, type=:letter>,
+ #<struct Greeb::Span from=11, to=12, type=:space>,
+ #<struct Greeb::Span from=12, to=14, type=:integer>,
+ #<struct Greeb::Span from=14, to=15, type=:punct>,
+ #<struct Greeb::Span from=15, to=16, type=:space>,
+ #<struct Greeb::Span from=16, to=18, type=:letter>,
+ #<struct Greeb::Span from=18, to=19, type=:space>,
+ #<struct Greeb::Span from=19, to=28, type=:letter>,
+ #<struct Greeb::Span from=28, to=29, type=:space>,
+ #<struct Greeb::Span from=29, to=35, type=:letter>,
+ #<struct Greeb::Span from=35, to=36, type=:space>,
+ #<struct Greeb::Span from=36, to=38, type=:letter>,
+ #<struct Greeb::Span from=38, to=39, type=:space>,
+ #<struct Greeb::Span from=39, to=44, type=:float>,
+ #<struct Greeb::Span from=44, to=47, type=:punct>,
+ #<struct Greeb::Span from=47, to=49, type=:break>,
+ #<struct Greeb::Span from=49, to=53, type=:letter>,
+ #<struct Greeb::Span from=53, to=54, type=:space>,
+ #<struct Greeb::Span from=54, to=59, type=:letter>,
+ #<struct Greeb::Span from=59, to=60, type=:space>,
+ #<struct Greeb::Span from=60, to=63, type=:letter>,
+ #<struct Greeb::Span from=63, to=64, type=:punct>,
+ #<struct Greeb::Span from=64, to=65, type=:break>]
 =end
 ```
 
@@ -99,8 +99,8 @@ text = 'Hello! How are you?'
 tokens = Greeb::Tokenizer.tokenize(text)
 pp Greeb::Segmentator.new(tokens).sentences
 =begin
-[#<struct Greeb::Entity from=0, to=6, type=:sentence>,
- #<struct Greeb::Entity from=7, to=19, type=:sentence>]
+[#<struct Greeb::Span from=0, to=6, type=:sentence>,
+ #<struct Greeb::Span from=7, to=19, type=:sentence>]
 =end
 ```
 
@@ -113,21 +113,21 @@ tokens = Greeb::Tokenizer.tokenize(text)
 segmentator = Greeb::Segmentator.new(tokens)
 pp segmentator.extract(segmentator.sentences)
 =begin
-{#<struct Greeb::Entity from=0, to=6, type=:sentence>=>
-  [#<struct Greeb::Entity from=0, to=5, type=:letter>,
-   #<struct Greeb::Entity from=5, to=6, type=:punct>],
- #<struct Greeb::Entity from=7, to=19, type=:sentence>=>
-  [#<struct Greeb::Entity from=7, to=10, type=:letter>,
-   #<struct Greeb::Entity from=10, to=11, type=:separ>,
-   #<struct Greeb::Entity from=11, to=14, type=:letter>,
-   #<struct Greeb::Entity from=14, to=15, type=:separ>,
-   #<struct Greeb::Entity from=15, to=18, type=:letter>,
-   #<struct Greeb::Entity from=18, to=19, type=:punct>]}
+{#<struct Greeb::Span from=0, to=6, type=:sentence>=>
+  [#<struct Greeb::Span from=0, to=5, type=:letter>,
+   #<struct Greeb::Span from=5, to=6, type=:punct>],
+ #<struct Greeb::Span from=7, to=19, type=:sentence>=>
+  [#<struct Greeb::Span from=7, to=10, type=:letter>,
+   #<struct Greeb::Span from=10, to=11, type=:space>,
+   #<struct Greeb::Span from=11, to=14, type=:letter>,
+   #<struct Greeb::Span from=14, to=15, type=:space>,
+   #<struct Greeb::Span from=15, to=18, type=:letter>,
+   #<struct Greeb::Span from=18, to=19, type=:punct>]}
 =end
 ```
 
 ### Parsing API
-Texts are often include some special entities such as URLs and e-mail
+Texts are often include some special spans such as URLs and e-mail
 addresses. Greeb can help you in these strings retrieval.
 
 #### URL and E-mail retrieval
@@ -136,12 +136,12 @@ text = 'My website is http://nlpub.ru and e-mail is example@example.com.'
 
 pp Greeb::Parser.urls(text).map { |e| [e, text[e.from...e.to]] }
 =begin
-[[#<struct Greeb::Entity from=14, to=29, type=:url>, "http://nlpub.ru"]]
+[[#<struct Greeb::Span from=14, to=29, type=:url>, "http://nlpub.ru"]]
 =end
 
 pp Greeb::Parser.emails(text).map { |e| [e, text[e.from...e.to]] }
 =begin
-[[#<struct Greeb::Entity from=44, to=63, type=:email>, "example@example.com"]]
+[[#<struct Greeb::Span from=44, to=63, type=:email>, "example@example.com"]]
 =end
 ```
 
@@ -153,7 +153,7 @@ text = 'Hello, G.L.H.F. everyone!'
 
 pp Greeb::Parser.abbrevs(text).map { |e| [e, text[e.from...e.to]] }
 =begin
-[[#<struct Greeb::Entity from=7, to=15, type=:abbrev>, "G.L.H.F."]]
+[[#<struct Greeb::Span from=7, to=15, type=:abbrev>, "G.L.H.F."]]
 =end
 ```
 
@@ -161,13 +161,13 @@ The algorithm is not so accurate, but still useful in many practical
 situations.
 
 ## Tokens
-Greeb operates with entities, tuples of *(from, to, kind)*, where
-*from* is a beginning of the entity, *to* is an ending of the entity,
-and *kind* is a type of the entity.
+Greeb operates with spans, tuples of *(from, to, kind)*, where
+*from* is a beginning of the span, *to* is an ending of the span,
+and *kind* is a type of the span.
 
-There are several entity types at the tokenization stage: `:letter`,
+There are several span types at the tokenization stage: `:letter`,
 `:float`, `:integer`, `:separ`, `:punct` (for punctuation), `:spunct`
-(for in-sentence punctuation), and `:break`.
+(for in-sentence punctuation), `:space`, and `:break`.
 
 ## Contributing
 1. Fork it;

data/bin/greeb

@@ -8,6 +8,6 @@ require 'greeb'
 
 text = STDIN.read.tap(&:chomp!)
 
-Greeb[text].each do |entity|
-  puts text[entity.from...entity.to] unless [:space, :break].include? entity.type
+Greeb[text].each do |span|
+  puts text[span.from...span.to] unless [:space, :break].include? span.type
 end

data/lib/greeb.rb

@@ -1,45 +1,8 @@
 # encoding: utf-8
 
 require 'greeb/version'
-
-# Greeb operates with entities, tuples of *(from, to, kind)*, where
-# *from* is a beginning of the entity, *to* is an ending of the entity,
-# and *kind* is a type of the entity.
-#
-# There are several entity types: `:letter`, `:float`, `:integer`,
-# `:separ` for separators, `:punct` for punctuation characters,
-# `:spunct` for in-sentence punctuation characters, and
-# `:break` for line endings.
-#
-class Greeb::Entity < Struct.new(:from, :to, :type)
-  # @private
-  def <=> other
-    if (comparison = self.from <=> other.from) == 0
-      self.to <=> other.to
-    else
-      comparison
-    end
-  end
-end
-
-# This runtime error appears when {Greeb::Tokenizer} or
-# {Greeb::Segmentator} tries to recognize unknown character.
-#
-class Greeb::UnknownEntity < RuntimeError
-  attr_reader :text, :pos
-
-  # @private
-  def initialize(text, pos)
-    @text, @pos = text, pos
-  end
-
-  # Generate the real error message.
-  #
-  def to_s
-    'Could not recognize character "%s" @ %d' % [text[pos], pos]
-  end
-end
-
+require 'greeb/exceptions'
+require 'greeb/span'
 require 'greeb/strscan'
 require 'greeb/tokenizer'
 require 'greeb/segmentator'

data/lib/greeb/core.rb

@@ -13,13 +13,13 @@ module Greeb::Core
   #
   # @param text [String] input text.
   #
-  # @return [Array<Greeb::Entity>] a set of tokens.
+  # @return [Array<Greeb::Span>] a set of tokens.
   #
-  def analyze text
+  def analyze(text, helpers = HELPERS)
     Greeb::Tokenizer.tokenize(text).tap do |tokens|
-      HELPERS.each do |helper|
+      helpers.each do |helper|
         Greeb::Parser.public_send(helper, text).each do |parsed|
-          extract_tokens(tokens, parsed)
+          extract_spans(tokens, parsed)
         end
       end
     end
@@ -28,17 +28,18 @@ module Greeb::Core
   alias_method :'[]', :analyze
 
   protected
-  # Extact tokens of the specified type from the input tokens set.
+  # Extact spans of the specified type from the input spans set.
   #
-  # @param tokens [Array<Greeb::Entity>] input tokens set.
-  # @param entity [Greeb::Entity] token to be extracted.
+  # @param spans [Array<Greeb::Span>] input spans set.
+  # @param span [Greeb::Span] span to be extracted.
   #
-  # @return [Greeb::Entity] token to be extracted.
+  # @return [Greeb::Span] span to be extracted.
   #
-  def extract_tokens(tokens, entity)
-    from = tokens.index { |e| e.from == entity.from }
-    to = tokens.index { |e| e.to == entity.to }
-    tokens[from..to] = entity
+  def extract_spans(spans, span)
+    from = spans.index { |e| e.from == span.from }
+    to = spans.index { |e| e.to == span.to }
+    return unless from && to
+    spans[from..to] = span
   end
 end
 

data/lib/greeb/exceptions.rb

@@ -0,0 +1,17 @@
+# This runtime error appears when {Greeb::Tokenizer} or
+# {Greeb::Segmentator} tries to recognize unknown character.
+#
+class Greeb::UnknownEntity < RuntimeError
+  attr_reader :text, :pos
+
+  # @private
+  def initialize(text, pos)
+    @text, @pos = text, pos
+  end
+
+  # Generate the real error message.
+  #
+  def to_s
+    'Could not recognize character "%s" @ %d' % [text[pos], pos]
+  end
+end

data/lib/greeb/parser.rb

@@ -16,12 +16,15 @@ module Greeb::Parser
   # Another horrible pattern. Now for abbreviations.
   ABBREV = /\b((-{0,1}\p{L}\.)*|(-{0,1}\p{L}\. )*)-{0,1}\p{L}\./i
 
+  # This pattern matches anything that looks like HTML. Or not.
+  HTML = /<(.*?)>/i
+
   # Recognize URLs in the input text. Actually, URL is obsolete standard
   # and this code should be rewritten to use the URI concept.
   #
   # @param text [String] input text.
   #
-  # @return [Array<Greeb::Entity>] found URLs.
+  # @return [Array<Greeb::Span>] found URLs.
   #
   def urls(text)
     scan(text, URL, :url)
@@ -31,7 +34,7 @@ module Greeb::Parser
   #
   # @param text [String] input text.
   #
-  # @return [Array<Greeb::Entity>] found e-mail addresses.
+  # @return [Array<Greeb::Span>] found e-mail addresses.
   #
   def emails(text)
     scan(text, EMAIL, :email)
@@ -41,27 +44,37 @@ module Greeb::Parser
   #
   # @param text [String] input text.
   #
-  # @return [Array<Greeb::Entity>] found abbreviations.
+  # @return [Array<Greeb::Span>] found abbreviations.
   #
   def abbrevs(text)
     scan(text, ABBREV, :abbrev)
   end
 
+  # Recognize HTML-alike entities in the input text.
+  #
+  # @param text [String] input text.
+  #
+  # @return [Array<Greeb::Span>] found HTML entities.
+  #
+  def html(text)
+    scan(text, HTML, :html)
+  end
+
   private
-  # Implementation of regexp-based {Greeb::Entity} scanner.
+  # Implementation of regexp-based {Greeb::Span} scanner.
   #
   # @param text [String] input text.
   # @param regexp [Regexp] regular expression to be used.
-  # @param type [Symbol] type field for the new {Greeb::Entity} instances.
+  # @param type [Symbol] type field for the new {Greeb::Span} instances.
   # @param offset [Fixnum] offset of the next match.
   #
-  # @return [Array<Greeb::Entity>] found entities.
+  # @return [Array<Greeb::Span>] found entities.
   #
   def scan(text, regexp, type, offset = 0)
     Array.new.tap do |matches|
       while text and md = text.match(regexp)
         start, stop = md.offset(0)
-        matches << Greeb::Entity.new(offset + start, offset + stop, type)
+        matches << Greeb::Span.new(offset + start, offset + stop, type)
         text, offset = text[stop..-1], offset + stop
       end
     end

data/lib/greeb/segmentator.rb

@@ -13,7 +13,7 @@ class Greeb::Segmentator
 
   # Create a new instance of {Greeb::Segmentator}.
   #
-  # @param tokens [Array<Greeb::Entity>] tokens from [Greeb::Tokenizer].
+  # @param tokens [Array<Greeb::Span>] tokens from [Greeb::Tokenizer].
   #
   def initialize(tokens)
     @tokens = tokens
@@ -21,62 +21,60 @@ class Greeb::Segmentator
 
   # Sentences memoization method.
   #
-  # @return [Array<Greeb::Entity>] a set of sentences.
+  # @return [Array<Greeb::Span>] a set of sentences.
   #
   def sentences
-    @sentences ||= detect_entities(new_sentence, [:punct])
+    @sentences ||= detect_spans(new_sentence, [:punct])
   end
 
   # Subsentences memoization method.
   #
-  # @return [Array<Greeb::Entity>] a set of subsentences.
+  # @return [Array<Greeb::Span>] a set of subsentences.
   #
   def subsentences
-    @subsentences ||= detect_entities(new_subsentence, [:punct, :spunct])
+    @subsentences ||= detect_spans(new_subsentence, [:punct, :spunct])
   end
 
   # Extract tokens from the set of sentences.
   #
-  # @param sentences [Array<Greeb::Entity>] a list of sentences.
+  # @param sentences [Array<Greeb::Span>] a list of sentences.
   #
-  # @return [Hash<Greeb::Entity, Array<Greeb::Entity>>] a hash with
+  # @return [Array<Greeb::Span, Array<Greeb::Span>>] a hash with
   #   sentences as keys and tokens arrays as values.
   #
   def extract(sentences, collection = tokens)
-    Hash[
-      sentences.map do |s|
-        [s, collection.select { |t| t.from >= s.from and t.to <= s.to }]
-      end
-    ]
+    sentences.map do |s|
+      [s, collection.select { |t| t.from >= s.from and t.to <= s.to }]
+    end
   end
 
   protected
-  # Implementation of the entity detection method.
+  # Implementation of the span detection method.
   #
-  # @param sample [Greeb::Entity] a sample of entity to be cloned in the
+  # @param sample [Greeb::Span] a sample of span to be cloned in the
   # process.
   # @param stop_marks [Array<Symbol>] an array that stores the
-  # correspondent stop marks of the necessary entities.
+  # correspondent stop marks of the necessary spans.
   #
-  # @return [Array<Greeb::Entity>] a set of entites.
+  # @return [Array<Greeb::Span>] a set of entites.
   #
-  def detect_entities(sample, stop_marks)
+  def detect_spans(sample, stop_marks)
     collection = []
 
-    rest = tokens.inject(sample.dup) do |entity, token|
-      next entity if sentence_aint_start? entity, token
-      entity.from = token.from unless entity.from
-      next entity if entity.to and entity.to > token.to
+    rest = tokens.inject(sample.dup) do |span, token|
+      next span if sentence_aint_start? span, token
+      span.from = token.from unless span.from
+      next span if span.to and span.to > token.to
 
       if stop_marks.include? token.type
-        entity.to = find_forward(tokens, token).to
-        collection << entity
-        entity = sample.dup
+        span.to = find_forward(tokens, token).to
+        collection << span
+        span = sample.dup
       elsif ![:separ, :space].include? token.type
-        entity.to = token.to
+        span.to = token.to
       end
 
-      entity
+      span
     end
 
     if rest.from && rest.to
@@ -88,42 +86,42 @@ class Greeb::Segmentator
 
   private
   # Check the possibility of starting a new sentence by the specified
-  # pair of entity and token.
+  # pair of span and token.
   #
-  # @param entity [Greeb::Entity] an entity to be checked.
-  # @param token [Greeb::Entity] an token to be checked.
+  # @param span [Greeb::Span] an span to be checked.
+  # @param token [Greeb::Span] an token to be checked.
   #
   # @return true or false.
   #
-  def sentence_aint_start?(entity, token)
-    !entity.from and SENTENCE_AINT_START.include? token.type
+  def sentence_aint_start?(span, token)
+    !span.from and SENTENCE_AINT_START.include? token.type
   end
 
   # Find a forwarding token that has another type.
   #
-  # @param collection [Array<Greeb::Entity>] array of possible tokens.
-  # @param sample [Greeb::Entity] a token that is treated as a sample.
+  # @param collection [Array<Greeb::Span>] array of possible tokens.
+  # @param sample [Greeb::Span] a token that is treated as a sample.
   #
-  # @return [Greeb::Entity] a forwarding token.
+  # @return [Greeb::Span] a forwarding token.
   #
   def find_forward(collection, sample)
     collection.select { |t| t.from >= sample.from }.
       inject(sample) { |r, t| t.type == sample.type ? t : (break r) }
   end
 
-  # Create a new instance of {Greeb::Entity} with `:sentence` type.
+  # Create a new instance of {Greeb::Span} with `:sentence` type.
   #
-  # @return [Greeb::Entity] a new entity instance.
+  # @return [Greeb::Span] a new span instance.
   #
   def new_sentence
-    Greeb::Entity.new(nil, nil, :sentence)
+    Greeb::Span.new(nil, nil, :sentence)
   end
 
-  # Create a new instance of {Greeb::Entity} with `:subsentence` type.
+  # Create a new instance of {Greeb::Span} with `:subsentence` type.
   #
-  # @return [Greeb::Entity] a new entity instance.
+  # @return [Greeb::Span] a new span instance.
   #
   def new_subsentence
-    Greeb::Entity.new(nil, nil, :subsentence)
+    Greeb::Span.new(nil, nil, :subsentence)
   end
 end