lemmatizer 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ffb812c39d3601bb86da6d6e44c8375cc8078bfb88089a8357c14c92473209a
-  data.tar.gz: fe6a030e331e8138231dfaf48b136bf5dc7e4e8be40a8457664bdd8bf2b8f6b5
+  metadata.gz: 622ecac5f6d2ffa86a231a0bf2a1689bc5f6746d53a388c9efc4041c30f2e151
+  data.tar.gz: 7886e4ed919288df6216495f2f0b047e6555c6008c36dc1233c19925fb244501
 SHA512:
-  metadata.gz: f2505986c5b0919c7691fbec86afc814df9567e23024f68e9c7f1daa0afd1485534fcd700aa131146d12f8d7dbfb95fbcbce798fc37e4408afd7976d077d750f
-  data.tar.gz: 3e9e7f6289d906b2e4f984ccf285e24024de933b7aa9065f99559f057392299a69696ab82706624781f780d925eb6f1565f629462ec9a1f98fadfc3bea8810ba
+  metadata.gz: de93d602e0a85badd29aa47f9fa4e2970fc3f2b4feb24e596be1921820c57cc3a3d162127bfff817e5b06094e25612be01bbe4f5751bad8278cbd7d6b7bad38c
+  data.tar.gz: d80dd24c7cb03de971f5c1e8bb1cf3a3a19bb41c0a5dfdf0a079c4b205ba524a360296d28370d3d72328cc0ba91fc67da5862552d11baf80efdd1d263d8b3353

data/README.md

@@ -51,14 +51,14 @@ Supplying with user dict
 # You can supply custom dict files consisting of lines in the format of <pos>\s+<form>\s+<lemma>.
 # The data in user supplied files overrides the preset data. Here's the sample. 
 
-# --- sample.dict.txt (don't include hash symbol on the left) ---
+# --- sample.dict1.txt (don't include hash symbol on the left) ---
 # adj   higher   high
 # adj   highest  high
 # noun  MacBooks MacBook
 # ---------------------------------------------------------------
 
-lem = Lemmatizer.new("sample.dict.txt")
-# => 3 lexical items added from dict file provided
+lem = Lemmatizer.new("sample.dict1.txt")
+# => 3 items added from dict file provided
 
 p lem.lemma("higher", :adj)     # => "high"
 p lem.lemma("highest", :adj)    # => "high"
@@ -67,14 +67,38 @@ p lem.lemma("MacBooks", :noun)  # => "MacBook"
 # The argument to Lemmatizer.new can be either of the following:
 # 1) a path string to a dict file (e.g. "/path/to/dict.txt")
 # 2) an array of paths to dict files (e.g. ["./dict/noun.txt", "./dict/verb.txt"])
+```
+
+Resolving abbreviations
+-----------
+```ruby
+# You can use 'abbr' tag in user dicts to resolve abbreviations in text.
+
+# --- sample.dict2.txt (don't include hash symbol on the left) ---
+# abbr  utexas   University of Texas
+# abbr  mit      Massachusetts Institute of Technology
+# ---------------------------------------------------------------
+
+# <NOTE>
+# 1. Expressions on the right (substitutes) can contain white spaces, 
+#    while expressions in the middle (words to be replaced) cannot.
+# 2. Double/Single quotations could be used with substitute expressions,
+#    but not with original expressions.
+
+lem = Lemmatizer.new("sample.dict2.txt")
+# => 2 items added from dict file provided
 
+p lem.lemma("utexas", :abbr) # => "University of Texas"
+p lem.lemma("mit", :abbr)    # => "Massachusetts Institute of Technology"
 ```
 
 Author
 ------
+
 * Yoichiro Hasebe <yohasebe@gmail.com>
 
 Thanks for assistance and contributions:
+
 * Vladimir Ivic <http://vladimirivic.com>
 
 License

data/lib/lemmatizer/lemmatizer.rb

@@ -1,3 +1,4 @@
+
 module Lemmatizer
   class Lemmatizer
     DATA_DIR = File.expand_path('..', File.dirname(__FILE__))
@@ -51,6 +52,8 @@ module Lemmatizer
       ],
       :adv =>  [
       ],
+      :abbr =>  [
+      ],
       :unknown => [
       ]
     }
@@ -77,7 +80,7 @@ module Lemmatizer
 
     def lemma(form, pos = nil)
       unless pos
-        [:verb, :noun, :adj, :adv].each do |p|
+        [:verb, :noun, :adj, :adv, :abbr].each do |p|
           result = lemma(form, p)
           return result unless result == form
         end
@@ -168,6 +171,8 @@ module Lemmatizer
         return :adj
       when "r", "adverb", "adv"
         return :adv
+      when "b", "abbrev", "abbr", "abr"
+        return :abbr
       else
         return :unknown
       end
@@ -178,15 +183,24 @@ module Lemmatizer
       open_file(dict) do |io|
         io.each_line do |line|
           # pos must be either n|v|r|a or noun|verb|adverb|adjective
-          p, w, s = line.split(/\s+/)
+          p, w, s = line.split(/\s+/, 3)
           pos = str_to_pos(p)
+          word = w
+          substitute = s.strip
+          if /\A\"(.*)\"\z/ =~ substitute
+            substitute = $1
+          end
+          if /\A\'(.*)\'\z/ =~ substitute
+            substitute = $1
+          end
+          next unless (pos && word && substitute)
           if @wordlists[pos]
-            @wordlists[pos][w] = s
+            @wordlists[pos][word] = substitute
             num_lex_added += 1
           end
         end
       end
-      puts "#{num_lex_added} items added from #{File.basename dict}"
+      # puts "#{num_lex_added} items added from #{File.basename dict}"
     end
   end
 end

data/lib/lemmatizer/version.rb

@@ -1,3 +1,3 @@
 module Lemmatizer
-  VERSION = '0.2.1'
+  VERSION = '0.2.2'
 end

data/spec/lemmatizer_spec.rb

@@ -7,8 +7,9 @@ describe 'Lemmatizer' do
     @lemmatizer = Lemmatizer.new
     user_data1 = File.join(File.dirname(__FILE__), "user.dict1.txt")
     user_data2 = File.join(File.dirname(__FILE__), "user.dict2.txt")
+    user_data3 = File.join(File.dirname(__FILE__), "user.dict3.txt")
     @lemmatizer_single_userdict = Lemmatizer.new(user_data1)
-    @lemmatizer_multiple_userdicts = Lemmatizer.new([user_data1, user_data2])
+    @lemmatizer_multiple_userdicts = Lemmatizer.new([user_data2, user_data3])
   end
 
   describe '#lemma' do
@@ -48,7 +49,7 @@ describe 'Lemmatizer' do
     end
 
     it 'gives a result when no pos is given' do
-      # Order: :verb, :noun, :adv, or :adj
+      # Order: :verb, :noun, :adv, :adj, or :abbr
       result_1 = @lemmatizer.lemma('plays')
       expect(result_1).to eq('play')
 
@@ -81,23 +82,36 @@ describe 'Lemmatizer' do
       # 'MacBooks' -> 'MacBook'
       result_u1 = @lemmatizer_single_userdict.lemma('MacBooks', :noun)
       expect(result_u1).to eq('MacBook')
-      result_u2 = @lemmatizer_single_userdict.lemma('crying', :verb)
-      expect(result_u2).to eq('cry')
+      # 'iPhones' -> 'iPhone'
+      result_u2 = @lemmatizer_single_userdict.lemma('iPhones', :noun)
+      expect(result_u2).to eq('iPhone')
     end
 
     it 'can load uder dicts that override presets' do
-      # 'MacBooks' -> 'MacBook'
-      result_ud1 = @lemmatizer_multiple_userdicts.lemma('MacBooks', :noun)
-      expect(result_ud1).to eq('MacBook')
       # 'higher' -> 'high'
-      result_ud2 = @lemmatizer_multiple_userdicts.lemma('higher', :adj)
-      expect(result_ud2).to eq('high')
-      # 'highest' -> 'high'
       result_ud3 = @lemmatizer_multiple_userdicts.lemma('higher')
       expect(result_ud3).to eq('high')
       # check if (unoverridden) preset data is kept intact
       result_ud4 = @lemmatizer_multiple_userdicts.lemma('crying', :verb)
       expect(result_ud4).to eq('cry')
+      # 'I'm' -> 'I am'
+      result_ud5 = @lemmatizer_multiple_userdicts.lemma("I'm", :abbr)
+      expect(result_ud5).to eq('I am')
+      # 'You're' -> 'you are'
+      result_ud6 = @lemmatizer_multiple_userdicts.lemma("You're", :abbr)
+      expect(result_ud6).to eq("you are")
+      # 'you're' -> 'you are'
+      result_ud7 = @lemmatizer_multiple_userdicts.lemma("you're", :abbr)
+      expect(result_ud7).to eq("you are")
+      # 'h2s' -> 'Hydrogen Sulphide'
+      result_ud8 = @lemmatizer_multiple_userdicts.lemma("h2s", :abbr)
+      expect(result_ud8).to eq("Hydrogen Sulphide")
+      # 'utexas' -> 'University of Texas'
+      result_ud9 = @lemmatizer_multiple_userdicts.lemma("utexas", :abbr)
+      expect(result_ud9).to eq("University of Texas")
+      # 'mit' -> 'Massachusetts Institute of Technology'
+      result_ud10 = @lemmatizer_multiple_userdicts.lemma("mit", :abbr)
+      expect(result_ud10).to eq("Massachusetts Institute of Technology")
     end
   end
 end

data/spec/user.dict3.txt

@@ -0,0 +1,6 @@
+abbr I'm      "I am"
+abbr you're   "you are"
+abbr You're   "you are"
+abbr h2s      "Hydrogen Sulphide"
+abbr utexas   University of Texas
+abbr mit      Massachusetts Institute of Technology

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lemmatizer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-16 00:00:00.000000000 Z
+date: 2019-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -55,6 +55,7 @@ files:
 - spec/spec_helper.rb
 - spec/user.dict1.txt
 - spec/user.dict2.txt
+- spec/user.dict3.txt
 homepage: http://github.com/yohasebe/lemmatizer
 licenses:
 - MIT
@@ -83,3 +84,4 @@ test_files:
 - spec/spec_helper.rb
 - spec/user.dict1.txt
 - spec/user.dict2.txt
+- spec/user.dict3.txt