lemmatizer 0.1.1 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YmRjOGYyZWE4ZWJjNWJjYWU3MzFkY2M5MjU1OTM4MzMxOGM1OGYwYw==
-  data.tar.gz: !binary |-
-    ZjRiZGQ1NjI1MzU2NTEyM2JmMzg0NGZiNDI2ZGRiMzExNmNlNDllNw==
+SHA256:
+  metadata.gz: 3962f3411b45a381c605ddd975f34bfad4055d8bc63bfdf385ed9341f395f5c4
+  data.tar.gz: 7dd90d196fda8c109f5847a7fafb0917f99c6f508dd0488cc33d60ced2c435e6
 SHA512:
-  metadata.gz: !binary |-
-    NTM0YThiNDVhYWVlYjZkNDZlZGNmMTg2OTYxMDE0ZDYwNWM0NWE5MGE2YjA5
-    ODZmYTM2YmE1MmM5MGJhODUzOWUxYmQzYTcwMzBhMmIxNmRiOTEwOGRmOWFk
-    YmNmNzc1YWI5ZDMzMDk2NDBhNmExNTUyZDgwYTJhZjFlOTZkN2Y=
-  data.tar.gz: !binary |-
-    ZmFlNWQ2OWYzYTBmMjM1MmVlOThlMWNlMTIwNjAwYjgwMzYxNWM0YmUzMThj
-    YjJiODA3NGNmOTk0MzQ4ZmY2YTc2ODM1YmJhMzgxOTQ1ZmEzNTY4ZDNkMDky
-    YTdmYTk4NDY5MzAzZjk2M2ZhY2RmOTJjZDQwMmY3ODE5N2ViOTY=
+  metadata.gz: eef23c892d9d9544637196fa61dde985fe440308157fd05f0202511723f3c84585b294cea6d827b4652a1473ff89f2b1f7a4b242aed17ae97c1700ffdee302e6
+  data.tar.gz: 1e95bb9907884803e9413251df611bd999d698baf2a6c709e5eb8f4ccbe9b272894ce179931c3609fa50e37fcac08f444cc10e05602312362b72f99fb871c77f

data/README.md

@@ -4,6 +4,8 @@ Lemmatizer for text in English.  Inspired by Python's [nltk.corpus.reader.wordne
 
 Based on code posted by mtbr at his blog entry [WordNet-based lemmatizer](http://d.hatena.ne.jp/mtbr/20090303/prfrnlprubyWordNetbasedlemmatizer)
 
+Version 0.2 has added functionality to add user supplied data at runtime 
+
 Installation
 ------------
     sudo gem install lemmatizer
@@ -42,6 +44,24 @@ p lem.lemma("higher", :adj) # => "higher" not "high"!
 # Modify dict/index.{noun|verb|adj|adv} if necessary.
 ```
 
+Supplying with user dict
+-----------
+```ruby
+# You can supply files with additional dict data consisting of lines in the format of <pos>\s+<form>\s+<lemma>.
+# The data in user supplied files overrides the preset data
+
+------ sample.dict.txt -----
+adj   higher   high
+adj   highest  high
+noun  MacBooks MacBook
+----------------------------
+
+lem = Lemmatizer.new("sample.dict.txt")
+p lem.lemma("higher", :adj)    # => "high"
+p lem.lemma("highest", :adj)   # => "high"
+p lem.lemma("MacBooks", :noun  # => "MacBook"
+```
+
 Author
 ------
 * Yoichiro Hasebe <yohasebe@gmail.com>
@@ -51,4 +71,4 @@ Thanks for assistance and contributions:
 
 License
 -------
-Licensed under the MIT license.
+Licensed under the MIT license.

data/Rakefile

@@ -3,7 +3,7 @@ require 'rspec/core'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec) do |spec|
-	spec.pattern = FileList['spec/**/*_spec.rb']
+  spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
-task :default => :spec
+task :default => :spec

data/lemmatizer.gemspec

@@ -1,5 +1,3 @@
-# -*- encoding: utf-8 -*-
-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 

data/lib/dict/cleanup.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+target = ARGV[0]
+base = File.basename(target)
+
+case base
+when /\.noun/
+  mode = :noun
+when /\.verb/
+  mode = :verb
+when /\.adj/
+  mode = :adj
+when /\.adv/
+  mode = :adv
+end
+  
+newtarget = base + "-mod"
+
+infile = File.open(target)
+lines = infile.readlines
+infile.close
+
+results = {}
+lines.each do |line|
+  /^([^\s]+)/ =~ line
+  case mode
+  when :noun
+    lemma = $1.sub(/s\z/, "").sub(/e\z/, "")
+  when :verb
+    lemma = $1.sub(/s\z/, "").sub(/d\z/, "").sub(/ing\z/, "").sub(/e\z/, "")
+  when :adj
+    lemma = $1.sub(/r\z/, "").sub(/st\z/, "").sub(/e\z/, "").sub(/i\z/, "")
+  when :adv
+    lemma = $1.sub(/r\z/, "").sub(/st\z/, "").sub(/e\z/, "").sub(/i\z/, "")
+  end
+  if results[lemma]
+    next
+  else
+    results[lemma] = line
+  end
+end
+
+outfile = File.open(newtarget, "w")
+outfile.write(results.values.join(""))
+outfile.close

data/lib/lemmatizer.rb

@@ -1,12 +1,10 @@
-# -*- coding: utf-8; mode: ruby -*-
-
 require 'stringio'
 require 'lemmatizer/version'
 require 'lemmatizer/core_ext'
 require 'lemmatizer/lemmatizer'
 
 module Lemmatizer
-  def self.new
-    Lemmatizer.new
+  def self.new(dict = nil)
+    Lemmatizer.new(dict)
   end
 end

data/lib/lemmatizer/core_ext.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8; mode: ruby -*-
-
 module Lematizer
   class ::String
     def endwith(s)

data/lib/lemmatizer/lemmatizer.rb

@@ -1,74 +1,74 @@
-# -*- coding: utf-8; mode: ruby -*-
-
 module Lemmatizer
   class Lemmatizer
     DATA_DIR = File.expand_path('..', File.dirname(__FILE__))
-    
+
     WN_FILES = {
       :noun => [
-        DATA_DIR + '/dict/index.noun', 
+        DATA_DIR + '/dict/index.noun',
         DATA_DIR + '/dict/noun.exc'
       ],
       :verb => [
-        DATA_DIR + '/dict/index.verb', 
+        DATA_DIR + '/dict/index.verb',
         DATA_DIR + '/dict/verb.exc'
       ],
       :adj  => [
-        DATA_DIR + '/dict/index.adj', 
+        DATA_DIR + '/dict/index.adj',
         DATA_DIR + '/dict/adj.exc'
       ],
       :adv  => [
-        DATA_DIR + '/dict/index.adv', 
+        DATA_DIR + '/dict/index.adv',
         DATA_DIR + '/dict/adv.exc'
       ]
     }
 
     MORPHOLOGICAL_SUBSTITUTIONS = {
       :noun => [
-        ['s',    ''   ], 
-        ['ses',  's'  ], 
-        ['ves',  'f'  ], 
+        ['s',    ''   ],
+        ['ses',  's'  ],
+        ['ves',  'f'  ],
         ['xes',  'x'  ],
-        ['zes',  'z'  ], 
-        ['ches', 'ch' ], 
+        ['zes',  'z'  ],
+        ['ches', 'ch' ],
         ['shes', 'sh' ],
-        ['men',  'man'], 
+        ['men',  'man'],
         ['ies',  'y'  ]
       ],
       :verb => [
-        ['s',   '' ], 
-        ['ies', 'y'], 
-        ['es',  'e'], 
+        ['s',   '' ],
+        ['ies', 'y'],
+        ['es',  'e'],
         ['es',  '' ],
-        ['ed',  'e'], 
-        ['ed',  '' ], 
-        ['ing', 'e'], 
+        ['ed',  'e'],
+        ['ed',  '' ],
+        ['ing', 'e'],
         ['ing', '' ]
       ],
       :adj =>  [
-        ['er',  '' ], 
-        ['est', '' ], 
-        ['er',  'e'], 
+        ['er',  '' ],
+        ['est', '' ],
+        ['er',  'e'],
         ['est', 'e']
       ],
       :adv =>  [
+      ],
+      :unknown => [
       ]
     }
 
-    def initialize(files = WN_FILES)
+    def initialize(dict = nil)
       @wordlists  = {}
       @exceptions = {}
-      
+
       MORPHOLOGICAL_SUBSTITUTIONS.keys.each do |x|
         @wordlists[x]  = {}
         @exceptions[x] = {}
       end
-
-      if files
-        files.each_pair do |pos, pair|
-          load_wordnet_files(pos, pair[0], pair[1])
-        end
+      
+      WN_FILES.each_pair do |pos, pair|
+        load_wordnet_files(pos, pair[0], pair[1])
       end
+
+      load_provided_dict(dict) if dict
     end
 
     def lemma(form, pos = nil)
@@ -79,15 +79,20 @@ module Lemmatizer
         end
 
         return form
-      end 
+      end
 
       each_lemma(form, pos) do |x|
         return x
       end
-      
+
       form
     end
-  
+
+    # Print object only on init
+    def inspect
+      "#{self}"
+    end
+
     private
 
     def open_file(*args)
@@ -142,11 +147,42 @@ module Lemmatizer
           yield x + 'ful'
         end
       else
-      
+
       each_substitutions(form, pos) do|x|
           yield x
         end
       end
     end
+
+    def str_to_pos(str)
+      case str
+      when "n", "noun"
+        return :noun
+      when "v", "verb"
+        return :noun
+      when "a", "j", "adjective", "adj"
+        return :adj
+      when "r", "adverb", "adv"
+        return :adv
+      else
+        return :unknown
+      end
+    end
+
+    def load_provided_dict(dict)
+      num_lex_added = 0
+      open_file(dict) do |io|
+        io.each_line do |line|
+          # pos must be either n|v|r|a or noun|verb|adverb|adjective
+          p, w, s = line.split(/\s+/)
+          pos = str_to_pos(p)
+          if @wordlists[pos]
+            @wordlists[pos][w] = s
+            num_lex_added += 1
+          end
+        end
+      end
+      puts "#{num_lex_added} lexical items added from dict file provided"
+    end
   end
 end

data/lib/lemmatizer/version.rb

@@ -1,5 +1,3 @@
-# -*- coding: utf-8; mode: ruby -*-
-
 module Lemmatizer
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/spec/lemmatizer_spec.rb

@@ -1,65 +1,93 @@
-# -*- coding: utf-8 -*-
-
 require 'spec_helper'
 require 'lemmatizer'
 
-describe "Lemmatizer" do
-	before do
-		@lemmatizer = Lemmatizer.new
-	end
-
-	describe "#lemma" do
-		it "takes a word form and its part-of-speech symbol (:noun, :verb, :adj, :adv) and then returns its lemma form" do
-			result_n1 = @lemmatizer.lemma("analyses", :noun)
-			result_n1.should == "analysis"
-
-      # Lemmatizer leaves alone words that its dictionary does not contain to keep proper names such as "James" intact.
-			result_n2 = @lemmatizer.lemma("MacBooks", :noun)
-			result_n2.should_not == "MacBook"
-
-			result_n3 = @lemmatizer.lemma("desks", :noun)
-			result_n3.should == "desk"
-
-			result_v1 = @lemmatizer.lemma("hired", :verb)
-			result_v1.should == "hire"
-
-			result_v2 = @lemmatizer.lemma("worried", :verb)
-			result_v2.should == "worry"
-
-			result_v3 = @lemmatizer.lemma("partying", :verb)
-			result_v3.should == "party"
-
-			result_a1 = @lemmatizer.lemma("better", :adj)
-			result_a1.should == "good"
-
-			result_a2 = @lemmatizer.lemma("hotter", :adj)
-			result_a2.should == "hot"
-
-			result_r1 = @lemmatizer.lemma("best", :adv)
-			result_r1.should == "well"
-
-			result_r2 = @lemmatizer.lemma("best", :adv)
-			result_r2.should_not == "good"
-      
-      # Lemmatizer give a result even when no pos is given, by assuming it to be :verb, :noun, :adv, or :adj.
-			result_1 = @lemmatizer.lemma("plays")
-			result_1.should == "play"
-
-			result_2 = @lemmatizer.lemma("oxen")
-			result_2.should == "ox"
-      
-			result_3 = @lemmatizer.lemma("higher")
-			result_3.should_not == "high" # since 'higher' is itself contained in the adj list.
-      
-			result_2 = @lemmatizer.lemma("asdfassda") # non-existing word
-			result_2.should == "asdfassda"
-      
-      # test cases for words used in README 
-			result_t1 = @lemmatizer.lemma("fired")
-			result_t1.should == "fire"
-
-			result_t2 = @lemmatizer.lemma("slower")
-			result_t2.should == "slow"      
-		end
-	end
+describe 'Lemmatizer' do
+
+  before(:all) do
+    @lemmatizer = Lemmatizer.new
+    user_data = File.join(File.dirname(__FILE__), "user.dict.txt")
+    @lemmatizer_with_userdata = Lemmatizer.new(user_data)
+  end
+
+  describe '#lemma' do
+    it 'takes a noun and returns its lemma' do
+      result_n1 = @lemmatizer.lemma('analyses', :noun)
+      expect(result_n1).to eq('analysis')
+
+      result_n3 = @lemmatizer.lemma('desks', :noun)
+      expect(result_n3).to eq('desk')
+    end
+
+    it 'takes a verb and returns its lemma' do
+      result_v1 = @lemmatizer.lemma('hired', :verb)
+      expect(result_v1).to eq('hire')
+
+      result_v2 = @lemmatizer.lemma('worried', :verb)
+      expect(result_v2).to eq('worry')
+
+      result_v3 = @lemmatizer.lemma('partying', :verb)
+      expect(result_v3).to eq('party')
+    end
+
+    it 'takes an adjective and returns its lemma' do
+      result_a1 = @lemmatizer.lemma('better', :adj)
+      expect(result_a1).to eq('good')
+
+      result_a2 = @lemmatizer.lemma('hotter', :adj)
+      expect(result_a2).to eq('hot')
+    end
+
+    it 'takes an adverb and returns its lemma' do
+      result_r1 = @lemmatizer.lemma('best', :adv)
+      expect(result_r1).to eq('well')
+
+      result_r2 = @lemmatizer.lemma('best', :adv)
+      expect(result_r2).not_to eq('good')
+    end
+
+    it 'gives a result when no pos is given' do
+      # Order: :verb, :noun, :adv, or :adj
+      result_1 = @lemmatizer.lemma('plays')
+      expect(result_1).to eq('play')
+
+      result_2 = @lemmatizer.lemma('oxen')
+      expect(result_2).to eq('ox')
+
+      # 'higher' is itself contained in the adj list.
+      result_3 = @lemmatizer.lemma('higher')
+      expect(result_3).not_to eq('high')
+
+      # Non-existing word
+      result_2 = @lemmatizer.lemma('asdfassda')
+      expect(result_2).to eq('asdfassda')
+
+      # Test cases for words used in README
+      result_t1 = @lemmatizer.lemma('fired')
+      expect(result_t1).to eq('fire')
+
+      result_t2 = @lemmatizer.lemma('slower')
+      expect(result_t2).to eq('slow')
+    end
+
+    it 'leaves alone words that dictionary does not contain' do
+      # Such as 'James' or 'MacBooks'
+      result_n2 = @lemmatizer.lemma('MacBooks', :noun)
+      expect(result_n2).not_to eq('MacBook')
+    end
+
+    it 'can load user dict that overrides presets' do
+      # 'MacBooks' -> 'MacBook'
+      result_u1 = @lemmatizer_with_userdata.lemma('MacBooks', :noun)
+      expect(result_u1).to eq('MacBook')
+      # 'higher' -> 'high'
+      result_u2 = @lemmatizer_with_userdata.lemma('higher', :adj)
+      expect(result_u2).to eq('high')
+      # 'highest' -> 'high'
+      result_u3 = @lemmatizer_with_userdata.lemma('higher')
+      expect(result_u3).to eq('high')
+      # check if (unoverridden) preset data is kept intact
+      result_u4 = @lemmatizer_with_userdata.lemma('crying', :verb)
+      expect(result_u4).to eq('cry')
+    end
+  end
 end

data/spec/user.dict.txt

@@ -0,0 +1,5 @@
+n MacBooks MacBook
+n iPhones iPhone
+n iPads iPad
+adj higher high
+adj highest high

metadata

@@ -1,30 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: lemmatizer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-03 00:00:00.000000000 Z
+date: 2019-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! "\n    Lemmatizer for text in English. Inspired by Python's nltk.corpus.reader.wordnet.morphy
+description: "\n    Lemmatizer for text in English. Inspired by Python's nltk.corpus.reader.wordnet.morphy
   package.\n  "
 email:
 - yohasebe@gmail.com
@@ -32,7 +32,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -40,6 +40,7 @@ files:
 - lemmatizer.gemspec
 - lib/dict/adj.exc
 - lib/dict/adv.exc
+- lib/dict/cleanup.rb
 - lib/dict/index.adj
 - lib/dict/index.adv
 - lib/dict/index.noun
@@ -52,6 +53,7 @@ files:
 - lib/lemmatizer/version.rb
 - spec/lemmatizer_spec.rb
 - spec/spec_helper.rb
+- spec/user.dict.txt
 homepage: http://github.com/yohasebe/lemmatizer
 licenses:
 - MIT
@@ -62,20 +64,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.1.9
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Englsh lemmatizer in Ruby
 test_files:
 - spec/lemmatizer_spec.rb
 - spec/spec_helper.rb
+- spec/user.dict.txt