engtagger 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 688eb41815586b529212a5451966f0cb2a1f4027c8aeb1c949fb437f1c8e5b4a
-  data.tar.gz: 1c7eea0dccc4c52133506220861c3ccce512d84f75df10060419dce5d24dfb8f
+  metadata.gz: 9c5c2c223bf80febf6a9130399b5d2a54dbdb043b0c9964289d98863d38572ce
+  data.tar.gz: 02ddd1ddee60893ffbd9ae686b382238ef99f6487b7eca38448cae39ec8f5c15
 SHA512:
-  metadata.gz: 3e5d673b7dd49cbe468c915e247f7d7e6b13002dc1b67e4e98c4e2d9066165f6cc6e0426d1393ff2711a4fb9df1294e44e9dbfc4c89e2b685b32a49482cd68c6
-  data.tar.gz: 609562c6c5cc2ea37762ff27e16a8689d569389eb9bf31af413d702e421a78d47acc8bdf993a1acffdce19631c565ca3452f371936f005e395db7f930933695e
+  metadata.gz: 91f5bfaee891d1c455e1a361ccd556714708d60a7c66e3ae0a24780cf2ce66c2fc975433da5b053453444701299b523254ba47bf89e62786368e6ae95300cc42
+  data.tar.gz: 8af44f088a545afe81b23009d77cd119949b27e027fdcee13344f68667768f2fd12a5eb89de189ae46ed5cc6f15a30acdcf2e07964b0efad0bc89664f79f1db0

data/.gitattributes

@@ -0,0 +1,4 @@
+# Packaging reads the working tree (gemspec uses `git ls-files`), so a
+# checkout that rewrites line endings would ship a corrupted model or lexicon.
+*.etm.gz binary
+*.hash binary

data/.gitignore

@@ -16,3 +16,6 @@ test/tmp
 test/version_tmp
 tmp
 /.idea
+experiments/
+
+.DS_Store

data/.rubocop.yml

@@ -3,7 +3,7 @@ AllCops:
   SuggestExtensions: false
   TargetRubyVersion: 2.6
 
-Documentation:
+Style/Documentation:
   Enabled: false
 
 Naming/AccessorMethodName:

data/CHANGELOG.md

@@ -1,5 +1,48 @@
 # Changelog
 
+## 0.4.4 (2026-08-14)
+
+This is the final planned release of the 0.4 line. Version 2.0 replaces the
+HMM engine with an averaged perceptron and does not ship the Penn
+Treebank-derived probability tables, so its tagging output differs. Pin
+`gem "engtagger", "~> 0.4"` to stay on this line; it will receive critical
+fixes only.
+
+- Fix the test suite silently rewriting the bundled lexicon files
+  (`lib/engtagger/*.hash`) on every run: a path-join bug in the test setup made
+  the existence check always fail, so `install` ran before each test. This was
+  also the enabler of the 0600-permission contamination behind #20. Tests are
+  ~6x faster as a result.
+- `get_sentences` now applies its intended punctuation cleanup. Four of its
+  five reattachment rules had discarded their results since the original port
+  (only the leading-backquote rule was ever applied), and their regexes were
+  corrupted by string escaping (`\W` matched a literal "W"). Sentences returned
+  by this method now read naturally (`"Lisa Raines, a lawyer,"`,`"$3"`).
+  Known limitation carried over from the tokenizer: contractions other than
+  possessives remain space-separated (`"I 'm"`, `"is n't"`).
+- Fix the public `WORD` constant, whose regex was corrupted by the same
+  escaping problem and matched only `<w>`-style tags.
+- Tag regexes produced by `EngTagger.get_ext` (NUM, NN, SEN, etc.) now end with
+  a real `\s*`, so they consume trailing whitespace including newlines instead
+  of spaces only.
+- `assign_tag` no longer raises `NoMethodError` for words absent from the
+  lexicon; it returns `""` so callers fall back to `"nn"` as designed.
+- `EngTagger::VERSION` is now defined at runtime (`engtagger/version` was not
+  required by the library itself).
+- Read and write the Marshal lexicon files in binary mode to avoid corruption
+  on Windows.
+- Remove the stray shebang and executable bit from `lib/engtagger.rb`, and the
+  obsolete `require "rubygems"`.
+- Clarify licensing: the LICENSE file had a stray MIT license appended after
+  the GPLv2 text. The library is licensed as GPL-2.0-or-later; the MIT tail has
+  been removed and the gemspec now declares the SPDX identifier.
+- Gemspec: use the https homepage URL, add `metadata` URIs, and drop the
+  obsolete `test_files`.
+- Fix a dead test (`text_get_ext` typo) that had never run and would have
+  failed if enabled.
+- Note in the README that 0.4.x carries Penn Treebank-derived probability
+  tables; 2.x is trained on openly licensed data instead.
+
 ## 0.4.3
 
 - Fix gem packaging that produced files with owner-only (0600) permissions,

data/Gemfile

@@ -5,5 +5,6 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake"
+gem "rubocop", require: false
 gem "sin_lru_redux"
 gem "test-unit"

data/LICENSE

@@ -338,25 +338,3 @@ proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
 library.  If this is what you want to do, use the GNU Library General
 Public License instead of this License.
-Copyright (c) 2012 Yoichiro Hasebe
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -174,4 +174,8 @@ Aaron Coburn, the author of the original Perl version.
 
 ## License
 
-This library is distributed under the GPL.  Please see the LICENSE file.
+This library is distributed under the GPL (GPL-2.0-or-later).  Please see the LICENSE file.
+
+The 0.4 line ships probability tables derived from the Penn Treebank. Version
+2.0 drops them in favour of a model trained on openly licensed corpora; if the
+provenance of the bundled data matters for your use, prefer 2.x.

data/engtagger.gemspec

@@ -7,16 +7,29 @@ Gem::Specification.new do |gem|
   gem.email         = ["yohasebe@gmail.com"]
   gem.summary       = "A probability based, corpus-trained English POS tagger"
   gem.description   = "A Ruby port of Perl Lingua::EN::Tagger, a probability based, corpus-trained tagger that assigns POS tags to English text based on a lookup dictionary and a set of probability values."
-  gem.homepage      = "http://github.com/yohasebe/engtagger"
-  gem.license       = "GPL"
+  gem.homepage      = "https://github.com/yohasebe/engtagger"
+  gem.license       = "GPL-2.0-or-later"
   gem.required_ruby_version = Gem::Requirement.new(">= 2.6")
   gem.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features|tools|docs)/}) }
   end
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "engtagger"
+  gem.metadata      = {
+    "source_code_uri" => "https://github.com/yohasebe/engtagger",
+    "changelog_uri" => "https://github.com/yohasebe/engtagger/blob/master/CHANGELOG.md",
+    "bug_tracker_uri" => "https://github.com/yohasebe/engtagger/issues"
+  }
   gem.require_paths = ["lib"]
   gem.version       = EngTagger::VERSION
   gem.add_dependency "sin_lru_redux"
+  gem.post_install_message = <<~MESSAGE
+    EngTagger 0.4.4 is the final planned release of the 0.4 line.
+
+    Version 2.0 replaces the HMM engine with an averaged perceptron trained on
+    openly licensed data. It is considerably more accurate, and its tagging
+    output differs, so treat it as a rewrite rather than an upgrade.
+
+    To stay on this line, pin it:  gem "engtagger", "~> 0.4"
+  MESSAGE
 end

data/lib/engtagger/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class EngTagger
-  VERSION = "0.4.3"
+  VERSION = "0.4.4"
 end

data/lib/engtagger.rb

@@ -1,10 +1,8 @@
-#!/usr/bin/env ruby
-
 # frozen_string_literal: true
 
-require "rubygems"
 require "lru_redux"
-require_relative "./engtagger/porter"
+require_relative "engtagger/version"
+require_relative "engtagger/porter"
 
 module BoundedSpaceMemoizable
   def memoize(method, max_cache_size = 100_000)
@@ -52,7 +50,7 @@ class EngTagger
   def self.get_ext(tag = nil)
     return nil unless tag
 
-    Regexp.new("<#{tag}>[^<]+</#{tag}>\s*")
+    Regexp.new("<#{tag}>[^<]+</#{tag}>\\s*")
   end
 
   # Regexps to match XML-style part-of-speech tags
@@ -66,7 +64,7 @@ class EngTagger
   PAREN = get_ext("[lr]rb")
   QUOT  = get_ext("ppr")
   SEN   = get_ext("pp")
-  WORD  = get_ext("\w+")
+  WORD  = get_ext("\\w+")
   VB    = get_ext("vb")
   VBG   = get_ext("vbg")
   VBD   = get_ext("vbd")
@@ -97,7 +95,7 @@ class EngTagger
     TAGS[tag] || tag
   end
 
-  # The folloging is to make a hash to convert a pos tag to its definition
+  # The following is to make a hash to convert a pos tag to its definition
   # used by the explain_tag method
   tags = [
     "CC",   "Conjunction, coordinating",
@@ -157,7 +155,7 @@ class EngTagger
   #    => (Boolean) Stem single words using Porter module
   # * :weight_noun_phrases
   #    => (Boolean) When returning occurrence counts for a noun phrase, multiply
-  #        the valuethe number of words in the NP.
+  #        the value by the number of words in the NP.
   # * :longest_noun_phrase
   #    => (Integer) Will ignore noun phrases longer than this threshold. This
   #        affects only the get_words() and get_nouns() methods.
@@ -200,7 +198,7 @@ class EngTagger
     @conf[:word_path] = DEFAULT_WORDPATH
     @conf[:tag_path] = DEFAULT_TAGPATH
     @conf[:debug] = false
-    # assuming that we start analyzing from the beginninga new sentence...
+    # assuming that we start analyzing from the beginning of a new sentence...
     @conf[:current_tag] = "pp"
     @conf.merge!(params) if params
     if !File.exist?(@conf[:word_path]) || !File.exist?(@conf[:tag_path])
@@ -208,10 +206,10 @@ class EngTagger
       @@hmm = {}
       @@lexicon = {}
     else
-      lexf = File.open(@conf[:word_path], "r")
+      lexf = File.open(@conf[:word_path], "rb")
       @@lexicon = Marshal.load(lexf)
       lexf.close
-      hmmf = File.open(@conf[:tag_path], "r")
+      hmmf = File.open(@conf[:tag_path], "rb")
       @@hmm = Marshal.load(hmmf)
       hmmf.close
     end
@@ -304,12 +302,16 @@ class EngTagger
     tagged.split(%r{</pp>}).each do |line|
       sentences << strip_tags(line)
     end
-    sentences = sentences.map do |sentence|
-      sentence.gsub(Regexp.new(" ('s?) ")) { $1 + " " }
-      sentence.gsub(Regexp.new(" (\W+) ")) { $1 + " " }
-      sentence.gsub(Regexp.new(" (`+) ")) { " " + $1 }
-      sentence.gsub(Regexp.new(" (\W+)$")) { $1 }
-      sentence.gsub(Regexp.new("^(`+) ")) { $1 }
+    sentences.map do |sentence|
+      # Reattach punctuation to the neighboring word: opening quotes/brackets
+      # and currency signs join the following word, everything else (commas,
+      # closing marks, possessives) joins the preceding word. (These gsub
+      # results used to be discarded, so none of this cleanup was applied.)
+      sentence.gsub(/ ('s?) /) { "#{$1} " }
+              .gsub(/ ([`(\[{$]+) /) { " #{$1}" }
+              .gsub(/ ([^\w\s`(\[{$]+) /) { "#{$1} " }
+              .gsub(/ ([^\w\s]+)\z/) { $1 }
+              .gsub(/\A([`(\[{$]+) /) { $1 }
     end
   end
 
@@ -360,7 +362,7 @@ class EngTagger
     build_matches_hash(build_trimmed(tagged, tags))
   end
 
-  # Returns all types of verbs and does not descriminate between the
+  # Returns all types of verbs and does not discriminate between the
   # various kinds. Combines all other verb methods listed in this
   # class.
   #
@@ -588,10 +590,10 @@ class EngTagger
     load_tags(@conf[:tag_lex])
     load_words(@conf[:word_lex])
     load_words(@conf[:unknown_lex])
-    File.open(@conf[:word_path], "w") do |f|
+    File.open(@conf[:word_path], "wb") do |f|
       Marshal.dump(@@lexicon, f)
     end
-    File.open(@conf[:tag_path], "w") do |f|
+    File.open(@conf[:tag_path], "wb") do |f|
       Marshal.dump(@@hmm, f)
     end
   end
@@ -635,7 +637,7 @@ class EngTagger
     @conf[:stem] ? word.stem : word
   end
 
-  # This method will reset the preceeding tag to a sentence ender (PP).
+  # This method will reset the preceding tag to a sentence ender (PP).
   # This prepares the first word of a new sentence to be tagged correctly.
   def reset
     @conf[:current_tag] = "pp"
@@ -705,7 +707,7 @@ class EngTagger
     end
     words = []
     tokenized.each_with_index do |_, i|
-      if tokenized[i + 1] && tokenized [i + 1] =~ /[A-Z\W]/ && tokenized[i] =~ /\A(.+)\.\z/
+      if tokenized[i + 1] && tokenized[i + 1] =~ /[A-Z\W]/ && tokenized[i] =~ /\A(.+)\.\z/
         w = $1
         # Don't separate the period off words that
         # meet any of the following conditions:
@@ -772,13 +774,13 @@ class EngTagger
       return "sym"
     end
     best_so_far = 0
-    w = @@lexicon[word]
+    w = @@lexicon[word] || {}
     t = @@hmm
 
     # TAG THE TEXT: What follows is a modified version of the Viterbi algorithm
     # which is used in most POS taggers
     best_tag = ""
-    t[prev_tag].each_key do |tag|
+    (t[prev_tag] || {}).each_key do |tag|
       # With @config[:relax] set, this method
       # will also include any `open classes' of POS tags
       pw = 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: engtagger
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -32,6 +32,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
 - ".gitignore"
 - ".rubocop.yml"
 - ".solargraph.yml"
@@ -50,10 +51,21 @@ files:
 - lib/engtagger/unknown.yml
 - lib/engtagger/version.rb
 - lib/engtagger/words.yml
-homepage: http://github.com/yohasebe/engtagger
+homepage: https://github.com/yohasebe/engtagger
 licenses:
-- GPL
-metadata: {}
+- GPL-2.0-or-later
+metadata:
+  source_code_uri: https://github.com/yohasebe/engtagger
+  changelog_uri: https://github.com/yohasebe/engtagger/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/yohasebe/engtagger/issues
+post_install_message: |
+  EngTagger 0.4.4 is the final planned release of the 0.4 line.
+
+  Version 2.0 replaces the HMM engine with an averaged perceptron trained on
+  openly licensed data. It is considerably more accurate, and its tagging
+  output differs, so treat it as a rewrite rather than an upgrade.
+
+  To stay on this line, pin it:  gem "engtagger", "~> 0.4"
 rdoc_options: []
 require_paths:
 - lib