punkt-segmenter 0.9.0 → 0.9.1

data/README.md

@@ -2,7 +2,7 @@
 
 This code is a ruby 1.9.x port of the Punkt sentence tokenizer algorithm implemented by the NLTK Project ([http://www.nltk.org/]). Punkt is a **language-independent**, unsupervised approach to **sentence boundary detection**. It is based on the assumption that a large number of ambiguities in the determination of sentence boundaries can be eliminated once abbreviations have been identified.
 
-The description of the algorithm is presented in the following academic paper:
+The full description of the algorithm is presented in the following academic paper:
 
 > Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection.  
 > Computational Linguistics 32: 485-525.  

data/lib/punkt-segmenter/frequency_distribution.rb

@@ -28,21 +28,18 @@ module Probability
     end
 
     def keys
-      result = @cache[:ordered_by_frequency_desc] || order_by_frequency_desc
-      result.map { |item| item[0] }
+      items.map { |item| item[0] }
     end
 
     def values
-      result = @cache[:ordered_by_frequency_desc] || order_by_frequency_desc
-      result.map { |item| item[1] }
+      items.map { |item| item[1] }
     end
 
     def items
-      @cache[:ordered_by_frequency_desc] || order_by_frequency_desc
+      @cache[:ordered_by_frequency_desc] ||= self.to_a.sort {|x,y| y[1] <=> x[1] }
     end
 
     def each(&block)
-      items = @cache[:ordered_by_frequency_desc] || order_by_frequency_desc
       items.each { |item| yield(item[0], item[1]) }
     end
 
@@ -111,11 +108,5 @@ module Probability
       self
     end
 
-  private
-  
-    def order_by_frequency_desc
-      @cache[:ordered_by_frequency_desc] = self.to_a.sort {|x,y| y[1] <=> x[1] }
-    end
-
   end
 end

data/lib/punkt-segmenter/punkt/base.rb

@@ -39,7 +39,7 @@ module Punkt
         if @language_vars.sent_end_chars.include?(tok)
           aug_token.sentence_break = true
         elsif aug_token.is_ellipsis?
-          aug_token.is_ellipsis = true
+          aug_token.ellipsis = true
         elsif aug_token.ends_with_period? && !tok.end_with?("..")
           tok_low = UnicodeUtils.downcase(tok.chop)
           if @parameters.abbreviation_types.include?(tok_low) || @parameters.abbreviation_types.include?(tok_low.split("-")[-1])

data/punkt-segmenter.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = "punkt-segmenter"
-  s.version       = "0.9.0"
+  s.version       = "0.9.1"
   s.platform      = Gem::Platform::RUBY
   s.summary       = "Ruby port of the NLTK Punkt sentence segmentation algorithm"
   s.require_paths = ['lib']
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   
   s.author        = "Luis Cipriani"
   s.email         = "lfcipriani@talleye.com"
-  s.homepage      = "http://github.com/lfcipriani/punkt-segmenter"
+  s.homepage      = "http://blog.talleye.com"
   
   s.add_dependency('unicode_utils', '>= 1.0.0')
   

data/test/punkt-segmenter/punkt/token_test.rb

@@ -117,5 +117,11 @@ class PunktTokenTest < Test::Unit::TestCase
     assert !token.is_non_punctuation?
   end
   
+  def test_to_s_and_inspect
+    token = Punkt::Token.new("foo", :abbr => true, :sentence_break => true, :ellipsis => true)
+    
+    assert_equal "<foo<A><E><S>>", token.inspect
+  end
+  
 end
 

data/test/test_helper.rb

@@ -13,4 +13,4 @@ require 'test/unit'
 require 'rubygems'
 require 'ruby-debug'
 
-require 'punkt-segmenter'
+require File.expand_path(File.dirname(__FILE__) + '/../lib/punkt-segmenter')

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: punkt-segmenter
 version: !ruby/object:Gem::Version 
-  hash: 59
   prerelease: false
   segments: 
   - 0
   - 9
-  - 0
-  version: 0.9.0
+  - 1
+  version: 0.9.1
 platform: ruby
 authors: 
 - Luis Cipriani
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-17 00:00:00 -03:00
+date: 2010-08-26 00:00:00 -03:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,7 +25,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
         segments: 
         - 1
         - 0
@@ -42,7 +40,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -56,7 +53,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -92,7 +88,7 @@ files:
 - punkt-segmenter.gemspec
 - script/console
 has_rdoc: true
-homepage: http://github.com/lfcipriani/punkt-segmenter
+homepage: http://blog.talleye.com
 licenses: []
 
 post_install_message: 
@@ -105,7 +101,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -114,7 +109,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"