punkt-segmenter 0.9.0

data/test/punkt-segmenter/punkt/language_vars_test.rb

@@ -0,0 +1,21 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+class PunktLanguageVarsTest < Test::Unit::TestCase
+
+  def setup
+    @lang_var = Punkt::LanguageVars.new
+    @sample = %Q{For example, the word "abbreviation" can itself be represented by the abbreviation abbr., abbrv. or abbrev.}
+  end
+
+  def test_word_tokenize
+    tokens = @lang_var.word_tokenize(@sample)
+
+    assert_equal 20  , tokens.size
+    assert_equal true, tokens.include?("abbr.")
+    assert_equal true, tokens.include?("\"")
+    assert_equal true, tokens.include?(",")
+    assert_equal true, tokens.include?("itself")
+  end
+
+end
+

data/test/punkt-segmenter/punkt/token_test.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+class PunktTokenTest < Test::Unit::TestCase
+
+  def test_token_properties_initialization
+    token = Punkt::Token.new("Test", :paragraph_start => true, 
+                                     :line_start => true, 
+                                     :sentence_break => true, 
+                                     :abbr => false)
+
+    assert_equal true,  token.paragraph_start
+    assert_equal true,  token.line_start
+    assert_equal true,  token.sentence_break
+    assert_equal false, token.abbr
+    assert_equal nil,   token.ellipsis
+  end
+  
+  def test_main_attributes
+    token = Punkt::Token.new("Test")
+    assert_equal "test", token.type
+
+    token = Punkt::Token.new("Test.")
+    assert_equal "test.", token.type
+    
+    token = Punkt::Token.new("Índico")
+    assert_equal "índico", token.type
+  end
+  
+  def test_type_without_period
+    token = Punkt::Token.new("Test")
+    assert_equal "test", token.type_without_period
+    
+    token = Punkt::Token.new("Test.")
+    assert_equal "test", token.type_without_period
+    
+    token = Punkt::Token.new("123.")
+    assert_equal "##number##", token.type_without_period
+  end
+  
+  def test_type_without_sentence_period
+    token = Punkt::Token.new("Test", :sentence_break => false)
+    assert_equal "test", token.type_without_sentence_period
+    
+    token = Punkt::Token.new("test.", :sentence_break => true)
+    assert_equal "test", token.type_without_sentence_period
+  end
+  
+  def test_first_upper?    
+    token = Punkt::Token.new("Test")
+    assert token.first_upper? 
+
+    token = Punkt::Token.new("Índico")
+    assert token.first_upper? 
+    
+    token = Punkt::Token.new("test.")
+    assert !token.first_upper?
+  end
+
+  def test_first_lower?    
+    token = Punkt::Token.new("Test")
+    assert !token.first_lower? 
+
+    token = Punkt::Token.new("Índico")
+    assert !token.first_lower? 
+    
+    token = Punkt::Token.new("test.")
+    assert token.first_lower?
+  end
+
+  def test_first_case    
+    token = Punkt::Token.new("Test")
+    assert_equal :upper, token.first_case
+
+    token = Punkt::Token.new("Índico")
+    assert_equal :upper, token.first_case
+
+    token = Punkt::Token.new("test.")
+    assert_equal :lower, token.first_case
+    
+    token = Punkt::Token.new("@")
+    assert_equal :none, token.first_case
+  end
+  
+  def test_is_ellipsis?
+    token = Punkt::Token.new("...")
+    assert token.is_ellipsis?
+
+    token = Punkt::Token.new("..")
+    assert token.is_ellipsis?
+
+    token = Punkt::Token.new("..foo")
+    assert !token.is_ellipsis?    
+  end
+
+  def test_is_initial?
+    token = Punkt::Token.new("C.")
+    assert token.is_initial?
+
+    token = Punkt::Token.new("B.M.")
+    assert !token.is_initial?
+  end
+
+  def test_is_alpha?
+    token = Punkt::Token.new("foo")
+    assert token.is_alpha?
+
+    token = Punkt::Token.new("!")
+    assert !token.is_alpha?    
+  end
+  
+  def test_is_non_punctuation?
+    token = Punkt::Token.new("foo")
+    assert token.is_non_punctuation?
+
+    token = Punkt::Token.new("!")
+    assert !token.is_non_punctuation?
+  end
+  
+end
+

data/test/punkt-segmenter/punkt/trainer_test.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+
+class PunktTrainerTest < Test::Unit::TestCase
+
+  def test_train_basic_portuguese_text_with_error
+    not_so_good_trainning_data = File.read(File.expand_path(File.dirname(__FILE__) + "/../../data/canudos.txt"))
+    
+    trainer = Punkt::Trainer.new()
+    trainer.train(not_so_good_trainning_data)
+    
+    parameters = trainer.parameters
+
+    # 'gol' is a word, not an abbreviation, the trainning text isn't good enough
+    assert parameters.abbreviation_types.include?("gol") 
+  end
+  
+  def test_improve_trainning_of_portuguese_text
+    not_so_good_trainning_data = File.read(File.expand_path(File.dirname(__FILE__) + "/../../data/canudos.txt"))
+    text_with_gol_as_a_word    = File.read(File.expand_path(File.dirname(__FILE__) + "/../../data/gripe.txt"))
+    
+    trainer = Punkt::Trainer.new()
+    trainer.train(not_so_good_trainning_data)
+    trainer.train(text_with_gol_as_a_word)
+    
+    parameters = trainer.parameters
+
+    # 'gol' is a word now, the trainning was better
+    assert !parameters.abbreviation_types.include?("gol")
+  end
+  
+end

data/test/punkt-segmenter/punkt_test.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+class PunktTest < Test::Unit::TestCase
+
+  def setup
+    @text      = File.read(File.expand_path(File.dirname(__FILE__) + "/../data/wikipedia_minute.txt"))
+    @tokenizer = Punkt::SentenceTokenizer.new(@text)
+  end
+
+  def test_sentences_as_string_indexes
+    result = @tokenizer.sentences_from_text(@text)
+    
+    assert_equal 7        , result.size #number of sentences
+    assert_equal [0,53]   , result.first
+    assert_equal [478,595], result.last
+  end
+  
+  def test_sentences_as_list_of_strings
+    result = @tokenizer.sentences_from_text(@text, :output => :sentences_text)
+    
+    assert_equal 7        , result.size #number of sentences
+    assert_equal result[0], "A minute is a unit of measurement of time or of angle."
+    assert_equal result[1], "The minute is a unit of time equal to 1/60th of an hour or 60 seconds by 1."
+    assert_equal result[2], "In the UTC time scale, a minute occasionally has 59 or 61 seconds; see leap second."
+    assert_equal result[3], "The minute is not an SI unit; however, it is accepted for use with SI units."
+    assert_equal result[4], "The symbol for minute or minutes is min."
+    assert_equal result[5], "The fact that an hour contains 60 minutes is probably due to influences from the Babylonians, who used a base-60 or sexagesimal counting system."
+    assert_equal result[6], "Colloquially, a min. may also refer to an indefinite amount of time substantially longer than the standardized length."
+  end
+  
+  def test_sentences_as_list_of_tokens
+    result = @tokenizer.sentences_from_text(@text, :output => :tokenized_sentences)
+    
+    assert_equal 7       , result.size #number of sentences
+    assert_equal "angle.", result.first.last
+    assert_equal 18      , result[1].size
+    assert_equal String  , result.last.first.class
+  end
+
+  def test_segment_list_of_tokens
+    list_of_tokens = Punkt::Base.new().tokenize_words(@text, :output => :string)
+    @tokenizer     = Punkt::SentenceTokenizer.new(@text)
+    result         = @tokenizer.sentences_from_tokens(list_of_tokens)
+    
+    assert_equal 7       , result.size #number of sentences
+    assert_equal "angle.", result.first.last
+    assert_equal 18      , result[1].size
+    assert_equal String  , result.last.first.class
+  end
+  
+  def test_realign_boundaries
+    text      = File.read(File.expand_path(File.dirname(__FILE__) + "/../data/canudos.txt"))    
+    tokenizer = Punkt::SentenceTokenizer.new(text)
+    result    = tokenizer.sentences_from_text(text, :output => :sentences_text)
+    
+    assert result[7].end_with?("(que vem bem ao nosso caso.")
+    assert result[8].start_with?(") \n\nDizem que durante")
+    
+    result    = tokenizer.sentences_from_text(text, :output => :sentences_text, :realign_boundaries => true)
+
+    assert result[7].end_with?("(que vem bem ao nosso caso.)")
+    assert result[8].start_with?("Dizem que durante")    
+  end
+
+end
+

data/test/test_helper.rb

@@ -0,0 +1,16 @@
+if ENV["coverage"]
+  require 'cover_me'
+  CoverMe.config do |c|
+      # where is your project's root:
+      c.project.root = File.expand_path(File.dirname(__FILE__) + '/..')
+
+      # what files are you interested in coverage for:
+      c.file_pattern = /(#{CoverMe.config.project.root}\/lib\/.+\.rb)/ix
+  end
+end
+
+require 'test/unit'
+require 'rubygems'
+require 'ruby-debug'
+
+require 'punkt-segmenter'

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification 
+name: punkt-segmenter
+version: !ruby/object:Gem::Version 
+  hash: 59
+  prerelease: false
+  segments: 
+  - 0
+  - 9
+  - 0
+  version: 0.9.0
+platform: ruby
+authors: 
+- Luis Cipriani
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-17 00:00:00 -03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: unicode_utils
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: cover_me
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug19
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: 
+email: lfcipriani@talleye.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/punkt-segmenter/frequency_distribution.rb
+- lib/punkt-segmenter/punkt/base.rb
+- lib/punkt-segmenter/punkt/language_vars.rb
+- lib/punkt-segmenter/punkt/parameters.rb
+- lib/punkt-segmenter/punkt/sentence_tokenizer.rb
+- lib/punkt-segmenter/punkt/token.rb
+- lib/punkt-segmenter/punkt/trainer.rb
+- lib/punkt-segmenter/punkt.rb
+- lib/punkt-segmenter.rb
+- README.md
+- LICENSE.txt
+- test/punkt-segmenter/frequency_distribution_test.rb
+- test/punkt-segmenter/punkt/language_vars_test.rb
+- test/punkt-segmenter/punkt/token_test.rb
+- test/punkt-segmenter/punkt/trainer_test.rb
+- test/punkt-segmenter/punkt_test.rb
+- test/test_helper.rb
+- Rakefile
+- punkt-segmenter.gemspec
+- script/console
+has_rdoc: true
+homepage: http://github.com/lfcipriani/punkt-segmenter
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Ruby port of the NLTK Punkt sentence segmentation algorithm
+test_files: []
+