RubyGems - engtagger - Versions diffs - 0.1.1 → 0.2.2 - Mend

engtagger 0.1.1 → 0.2.2

Files changed (17) hide show

data/test/test_engtagger.rb CHANGED Viewed

@@ -21,7 +21,7 @@ EOD
     @tagger = EngTagger.new
     tagpath = File.join($ENGTAGGER_LIB, @tagger.conf[:tag_path])
     wordpath = File.join($ENGTAGGER_LIB, @tagger.conf[:word_path])
-    if !File.exists?(tagpath) or !File.exists?(wordpath)
+    if !File.exist?(tagpath) or !File.exists?(wordpath)
       @tagger.install
     end
   end
@@ -69,7 +69,7 @@ EOD
   def test_clean_text
     test = "<html><body>I am <b>100% sure</b> that Dr. Watson is too naive. I'm sorry.</body></html>"
     model = ["I","am","100","%","sure","that","Dr.","Watson","is","too","naive",".","I","'m","sorry","."]
-    assert_equal(model, @tagger.clean_text(test))
+    assert_equal(model, @tagger.clean_text(test)) unless $no_hpricot
   end
   def test_clean_word
@@ -102,6 +102,38 @@ EOD
     assert_instance_of(Hash, result)
   end
+  def test_get_verbs
+    expected_result = { "have" => 1, "ruled" => 1, "contends" => 1 }
+    result = @tagger.get_verbs(@@tagged)
+    assert_equal(expected_result, result)
+  end
+  def test_get_adverbs
+    expected_result = { "otherwise" => 1 }
+    result = @tagger.get_adverbs(@@tagged)
+    assert_equal(expected_result, result)
+  end
+  def test_get_interrogatives
+    tagged = "<wdt>Which</wdt> <ppc>,</ppc> <wdt>whatever</wdt> <ppc>,</ppc> <wp>who</wp> <ppc>,</ppc> <wp>whoever</wp> <ppc>,</ppc> <wrb>when</wrb> <cc>and</cc> <wrb>how</wrb> <vbp>are</vbp> <det>all</det> <nns>examples</nns> <in>of</in> <nns>interrogatives</nns>"
+    expected_result = {"when"=>1, "how"=>1, "Which"=>1, "whatever"=>1, "who"=>1, "whoever"=>1}
+    result = @tagger.get_interrogatives(tagged)
+    assert_equal(expected_result, result)
+  end
+  def test_get_question_parts
+    tagged = "<wdt>Which</wdt> <ppc>,</ppc> <wdt>whatever</wdt> <ppc>,</ppc> <wp>who</wp> <ppc>,</ppc> <wp>whoever</wp> <ppc>,</ppc> <wrb>when</wrb> <cc>and</cc> <wrb>how</wrb> <vbp>are</vbp> <det>all</det> <nns>examples</nns> <in>of</in> <nns>interrogatives</nns>"
+    expected_result = {"when"=>1, "how"=>1, "Which"=>1, "whatever"=>1, "who"=>1, "whoever"=>1}
+    result = @tagger.get_question_parts(tagged)
+    assert_equal(expected_result, result)
+  end
+  def test_get_conjunctions
+    expected_result = { "and" => 2, "of" => 2, "for" => 1, "that" => 1, "in" => 1 }
+    result = @tagger.get_conjunctions(@@tagged)
+    assert_equal(expected_result, result)
+  end
   def test_get_proper_nouns
     test = "<nnp>BBC</nnp> <vbz>means</vbz> <nnp>British Broadcasting Corporation</nnp> <pp>.</pp>"
     result = @tagger.get_proper_nouns(test)
@@ -191,6 +223,11 @@ EOD
     text = ""
     assert(!@tagger.valid_text(text))
   end
+  def test_override_default_params
+    @tagger = EngTagger.new(:longest_noun_phrase => 3)
+    assert_equal 3, @tagger.conf[:longest_noun_phrase]
+  end
 end
 # Number of errors detected: 24

metadata CHANGED Viewed

@@ -1,86 +1,60 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification
 name: engtagger
-version: !ruby/object:Gem::Version
-  version: 0.1.1
+version: !ruby/object:Gem::Version
+  version: 0.2.2
 platform: ruby
-authors:
+authors:
 - Yoichiro Hasebe
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2008-05-15 00:00:00 +09:00
-default_executable:
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: hpricot
-  version_requirement:
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: "0"
-    version:
-- !ruby/object:Gem::Dependency
-  name: hoe
-  version_requirement:
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.1
-    version:
-description: A Ruby port of Perl Lingua::EN::Tagger, a probability based, corpus-trained  tagger that assigns POS tags to English text based on a lookup dictionary and  a set of probability values. The tagger assigns appropriate tags based on  conditional probabilities--it examines the preceding tag to determine the  appropriate tag for the current word. Unknown words are classified according to  word morphology or can be set to be treated as nouns or other parts of speech.   The tagger also extracts as many nouns and noun phrases as it can, using a set  of regular expressions.
-email: yohasebe@gmail.com
+date: 2022-02-03 00:00:00.000000000 Z
+dependencies: []
+description: A Ruby port of Perl Lingua::EN::Tagger, a probability based, corpus-trained
+  tagger that assigns POS tags to English text based on a lookup dictionary and a
+  set of probability values.
+email:
+- yohasebe@gmail.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- History.txt
-- LICENSE.txt
-- Manifest.txt
-- README.txt
-files:
-- History.txt
-- LICENSE.txt
-- Manifest.txt
-- README.txt
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
 - Rakefile
+- engtagger.gemspec
 - lib/engtagger.rb
 - lib/engtagger/porter.rb
 - lib/engtagger/pos_tags.hash
 - lib/engtagger/pos_words.hash
 - lib/engtagger/tags.yml
 - lib/engtagger/unknown.yml
+- lib/engtagger/version.rb
 - lib/engtagger/words.yml
 - test/test_engtagger.rb
-has_rdoc: true
-homepage: http://engtagger.rubyforge.org
-post_install_message:
-rdoc_options:
-- --main
-- README.txt
-require_paths:
+homepage: http://github.com/yohasebe/engtagger
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
-  requirements:
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version
-      version: "0"
-  version:
-required_rubygems_version: !ruby/object:Gem::Requirement
-  requirements:
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version
-      version: "0"
-  version:
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-rubyforge_project: engtagger
-rubygems_version: 1.1.1
-signing_key:
-specification_version: 2
-summary: English Part-of-Speech Tagger Library; a Ruby port of Lingua::EN::Tagger
-test_files:
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: A probability based, corpus-trained English POS tagger
+test_files:
 - test/test_engtagger.rb

data/History.txt DELETED Viewed

@@ -1,10 +0,0 @@
-=== 0.1.0 / 2008-05-14
-* Modified Synopsis section of Readme.txt
-* Created a description of tag set in Readme.txt
-* Fixed a few minor bugs
-=== 0.1.0 / 2008-05-06
-* Initial release
-* Functionalities are basically the same as those of Perl Lingua::EN::Tagger.

data/Manifest.txt DELETED Viewed

@@ -1,13 +0,0 @@
-History.txt
-LICENSE.txt
-Manifest.txt
-README.txt
-Rakefile
-lib/engtagger.rb
-lib/engtagger/porter.rb
-lib/engtagger/pos_tags.hash
-lib/engtagger/pos_words.hash
-lib/engtagger/tags.yml
-lib/engtagger/unknown.yml
-lib/engtagger/words.yml
-test/test_engtagger.rb

data/README.txt DELETED Viewed

@@ -1,140 +0,0 @@
-= EngTagger
-English Part-of-Speech Tagger Library; a Ruby port of Lingua::EN::Tagger
-=== Description
-A Ruby port of Perl Lingua::EN::Tagger, a probability based, corpus-trained
-tagger that assigns POS tags to English text based on a lookup dictionary and
-a set of probability values. The tagger assigns appropriate tags based on
-conditional probabilities--it examines the preceding tag to determine the
-appropriate tag for the current word. Unknown words are classified according to
-word morphology or can be set to be treated as nouns or other parts of speech.
-The tagger also extracts as many nouns and noun phrases as it can, using a set
-of regular expressions.
-=== Features
-* Assigns POS tags to English text
-* Extract noun phrases from tagged text
-* etc.
-=== Synopsis:
-  require 'rubygems'
-  require 'engtagger'
-  # Create a parser object
-  tgr = EngTagger.new
-  # Sample text
-  text = "Alice chased the big fat cat."
-  # Add part-of-speech tags to text
-  tagged = tgr.add_tags(text)
-  #=> "<nnp>Alice</nnp> <vbd>chased</vbd> <det>the</det> <jj>big</jj> <jj>fat</jj><nn>cat</nn> <pp>.</pp>"
-  # Get a list of all nouns and noun phrases with occurrence counts
-  word_list = tgr.get_words(text)
-  #=> {"Alice"=>1, "cat"=>1, "fat cat"=>1, "big fat cat"=>1}
-  # Get a readable version of the tagged text
-  readable = tgr.get_readable(text)
-  #=> "Alice/NNP chased/VBD the/DET big/JJ fat/JJ cat/NN ./PP"
-  # Get all nouns from a tagged output
-  nouns = tgr.get_nouns(tagged)
-  #=> {"cat"=>1, "Alice"=>1}
-  # Get all proper nouns
-  proper = tgr.get_proper_nouns(tagged)
-  #=> {"Alice"=>1}
-  # Get all noun phrases of any syntactic level
-  # (same as word_list but take a tagged input)
-  nps = tgr.get_noun_phrases(tagged)
-  #=> {"Alice"=>1, "cat"=>1, "fat cat"=>1, "big fat cat"=>1}
-=== Tag Set
-The set of POS tags used here is a modified version of the Penn Treebank tagset. Tags with non-letter characters have been redefined to work better in our data structures. Also, the "Determiner" tag (DET) has been changed from 'DT', in order to avoid confusion with the HTML tag, <DT>.
-  CC      Conjunction, coordinating               and, or
-  CD      Adjective, cardinal number              3, fifteen
-  DET     Determiner                              this, each, some
-  EX      Pronoun, existential there              there
-  FW      Foreign words
-  IN      Preposition / Conjunction               for, of, although, that
-  JJ      Adjective                               happy, bad
-  JJR     Adjective, comparative                  happier, worse
-  JJS     Adjective, superlative                  happiest, worst
-  LS      Symbol, list item                       A, A.
-  MD      Verb, modal                             can, could, 'll
-  NN      Noun                                    aircraft, data
-  NNP     Noun, proper                            London, Michael
-  NNPS    Noun, proper, plural                    Australians, Methodists
-  NNS     Noun, plural                            women, books
-  PDT     Determiner, prequalifier                quite, all, half
-  POS     Possessive                              's, '
-  PRP     Determiner, possessive second           mine, yours
-  PRPS    Determiner, possessive                  their, your
-  RB      Adverb                                  often, not, very, here
-  RBR     Adverb, comparative                     faster
-  RBS     Adverb, superlative                     fastest
-  RP      Adverb, particle                        up, off, out
-  SYM     Symbol                                  *
-  TO      Preposition                             to
-  UH      Interjection                            oh, yes, mmm
-  VB      Verb, infinitive                        take, live
-  VBD     Verb, past tense                        took, lived
-  VBG     Verb, gerund                            taking, living
-  VBN     Verb, past/passive participle           taken, lived
-  VBP     Verb, base present form                 take, live
-  VBZ     Verb, present 3SG -s form               takes, lives
-  WDT     Determiner, question                    which, whatever
-  WP      Pronoun, question                       who, whoever
-  WPS     Determiner, possessive & question       whose
-  WRB     Adverb, question                        when, how, however
-  PP      Punctuation, sentence ender             ., !, ?
-  PPC     Punctuation, comma                      ,
-  PPD     Punctuation, dollar sign                $
-  PPL     Punctuation, quotation mark left        ``
-  PPR     Punctuation, quotation mark right       ''
-  PPS     Punctuation, colon, semicolon, elipsis  :, ..., -
-  LRB     Punctuation, left bracket               (, {, [
-  RRB     Punctuation, right bracket              ), }, ]
-=== Requirements
-* Ruby 1.8.6
-* Hpricot[http://code.whytheluckystiff.net/hpricot/] (optional)
-=== Install
-  (sudo) gem install engtagger
-=== Author
-of this Ruby library
-* Yoichiro Hasebe (yohasebe [at] gmail.com)
-of the original Perl module
-* Aaron Coburn (acoburn [at] middlebury.edu)
-=== Acknowledgement
-This Ruby library is a direct port of Lingua::EN::Tagger available at CPAN.
-The credit for the crucial part of its algorithm/design therefore goes to
-Aaron Coburn, the author of the original Perl version.
-=== License
-This library is distributed under the GPL.  Please see the LICENSE file.