nhkore 0.3.7 → 0.3.8

data/lib/nhkore/variator.rb

@@ -1,46 +1,34 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class Variator
     def begin_variate(str)
       return str
     end
-    
+
     def variate(str)
       str = begin_variate(str)
       str = end_variate(str)
-      
+
       return str
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BasicVariator < Variator
@@ -48,38 +36,38 @@ module NHKore
       return [] # No variations; don't return nil
     end
   end
-  
+
   ###
   # Guesses a word's dictionary/plain form (辞書形).
-  # 
+  #
   # It doesn't work very well,but better than nothing...
-  # 
+  #
   # @since  0.2.0
   ###
   class DictFormVariator < Variator
     attr_accessor :deinflector
-    
+
     def initialize(*)
       require 'set' # Must require manually because JapaneseDeinflector is old
       require 'japanese_deinflector'
-      
+
       super
-      
-      @deinflector = JapaneseDeinflector.new()
+
+      @deinflector = JapaneseDeinflector.new
     end
-    
+
     def end_variate(str)
       guess = @deinflector.deinflect(str)
-      
+
       return [] if guess.length < 1
       return [] if (guess = guess[0])[:weight] < 0.5
-      
+
       return [guess[:word]]
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BestVariator < DictFormVariator

data/lib/nhkore/version.rb

@@ -1,26 +1,14 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
 module NHKore
-  VERSION = '0.3.7'
+  VERSION = '0.3.8'
 end

data/lib/nhkore/word.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -29,7 +17,7 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.1.0
   ###
   class Word
@@ -39,42 +27,42 @@ module NHKore
     attr_reader :kana
     attr_reader :kanji
     attr_reader :key
-    
+
     def initialize(defn: nil,eng: nil,freq: 1,kana: nil,kanji: nil,unknown: nil,word: nil,**kargs)
       super()
-      
-      if !word.nil?()
-        defn = word.defn if defn.nil?()
-        eng = word.eng if eng.nil?()
-        freq = word.freq if freq.nil?()
-        kana = word.kana if kana.nil?()
-        kanji = word.kanji if kanji.nil?()
+
+      if !word.nil?
+        defn = word.defn if defn.nil?
+        eng = word.eng if eng.nil?
+        freq = word.freq if freq.nil?
+        kana = word.kana if kana.nil?
+        kanji = word.kanji if kanji.nil?
       end
-      
+
       raise ArgumentError,"freq[#{freq}] cannot be < 1" if freq < 1
-      
-      if !unknown.nil?()
+
+      if !unknown.nil?
         # kanji?() only tests if it contains kanji, so don't use kana?().
         if Util.kanji?(unknown)
           if !Util.empty_web_str?(kanji)
             raise ArgumentError,"unknown[#{unknown}] will overwrite kanji[#{kanji}]"
           end
-          
+
           kanji = unknown
         else
           if !Util.empty_web_str?(kana)
             raise ArgumentError,"unknown[#{unknown}] will overwrite kana[#{kana}]"
           end
-          
+
           kana = unknown
         end
       end
-      
+
       kana = nil if Util.empty_web_str?(kana)
       kanji = nil if Util.empty_web_str?(kanji)
-      
-      raise ArgumentError,'kanji and kana cannot both be empty' if kana.nil?() && kanji.nil?()
-      
+
+      raise ArgumentError,'kanji and kana cannot both be empty' if kana.nil? && kanji.nil?
+
       @defn = defn
       @eng = eng
       @freq = freq
@@ -82,118 +70,132 @@ module NHKore
       @kanji = kanji
       @key = "#{kanji}=#{kana}" # nil.to_s() is ''
     end
-    
+
     def encode_with(coder)
       # Ignore @key because it will be the key in the YAML/Hash.
       # Order matters.
-      
+
       coder[:kanji] = @kanji
       coder[:kana] = @kana
       coder[:freq] = @freq
       coder[:defn] = @defn
       coder[:eng] = @eng
     end
-    
+
     def self.load_data(key,hash)
-      key = key.to_s() # Change from a symbol
-      
+      key = key.to_s # Change from a symbol
+
       word = Word.new(
         defn: hash[:defn],
         eng: hash[:eng],
         kana: hash[:kana],
         kanji: hash[:kanji]
       )
-      
+
       if key != word.key
         raise ArgumentError,"the key from the hash[#{key}] does not match the generated key[#{word.key}]"
       end
-      
-      freq = hash[:freq].to_i() # nil.to_i() is 0
+
+      freq = hash[:freq].to_i # nil.to_i() is 0
       word.freq = freq if freq > 0
-      
+
       return word
     end
-    
+
     # Do not clean and/or strip spaces, as the raw text is important for
-    #   Defn and ArticleScraper.
+    # Defn and ArticleScraper.
+    #
+    # This originally only scraped 1 word, but multiple words were added
+    # after seeing this link for 産業能率大学, which is valid HTML:
+    #   https://www3.nhk.or.jp/news/easy/k10012759201000/k10012759201000.html
+    #
+    # @return [Array<Word>] the scraped {Word}(s)
     def self.scrape_ruby_tag(tag,missingno: nil,url: nil)
       # First, try <rb> tags.
-      kanji = tag.css('rb')
+      kanjis = tag.css('rb')
       # Second, try text nodes.
-      kanji = tag.search('./text()') if kanji.length < 1
+      kanjis = tag.search('./text()') if kanjis.length < 1
       # Third, try non-<rt> tags, in case of being surrounded by <span>, <b>, etc.
-      kanji = tag.search("./*[not(name()='rt')]") if kanji.length < 1
-      
-      raise ScrapeError,"no kanji at URL[#{url}] in tag[#{tag}]" if kanji.length < 1
-      raise ScrapeError,"too many kanji at URL[#{url}] in tag[#{tag}]" if kanji.length > 1
-      
-      kanji = kanji[0].text
-      kana = tag.css('rt')
-      
-      raise ScrapeError,"no kana at URL[#{url}] in tag[#{tag}]" if kana.length < 1
-      raise ScrapeError,"too many kana at URL[#{url}] in tag[#{tag}]" if kana.length > 1
-      
-      kana = kana[0].text
-      
-      if !missingno.nil?()
-        # Check kana first, since this is the typical scenario.
-        # - https://www3.nhk.or.jp/news/easy/k10012331311000/k10012331311000.html
-        # - '窓' in '（８）窓を開けて外の空気を入れましょう'
-        if Util.empty_web_str?(kana)
-          kana = missingno.kana_from_kanji(kanji)
-          
-          if !Util.empty_web_str?(kana)
-            Util.warn("using missingno for kana[#{kana}] from kanji[#{kanji}]")
-          end
-        elsif Util.empty_web_str?(kanji)
-          kanji = missingno.kanji_from_kana(kana)
-          
-          if !Util.empty_web_str?(kanji)
-            Util.warn("using missingno for kanji[#{kanji}] from kana[#{kana}]")
+      kanjis = tag.search("./*[not(name()='rt')]") if kanjis.length < 1
+
+      kanas = tag.css('rt')
+
+      raise ScrapeError,"no kanji at URL[#{url}] in tag[#{tag}]" if kanjis.length < 1
+      raise ScrapeError,"no kana at URL[#{url}] in tag[#{tag}]" if kanas.length < 1
+
+      if kanjis.length != kanas.length
+        raise ScrapeError,"number of kanji & kana mismatch at URL[#{url}] in tag[#{tag}]"
+      end
+
+      words = []
+
+      (0...kanjis.length).each do |i|
+        kanji = kanjis[i].text
+        kana = kanas[i].text
+
+        # Uncomment for debugging; really need a logger.
+        #puts "Word[#{i}]: #{kanji} => #{kana}"
+
+        if !missingno.nil?
+          # Check kana first, since this is the typical scenario.
+          # - https://www3.nhk.or.jp/news/easy/k10012331311000/k10012331311000.html
+          # - '窓' in '（８）窓を開けて外の空気を入れましょう'
+          if Util.empty_web_str?(kana)
+            kana = missingno.kana_from_kanji(kanji)
+
+            if !Util.empty_web_str?(kana)
+              Util.warn("using missingno for kana[#{kana}] from kanji[#{kanji}]")
+            end
+          elsif Util.empty_web_str?(kanji)
+            kanji = missingno.kanji_from_kana(kana)
+
+            if !Util.empty_web_str?(kanji)
+              Util.warn("using missingno for kanji[#{kanji}] from kana[#{kana}]")
+            end
           end
         end
+
+        raise ScrapeError,"empty kanji at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kanji)
+        raise ScrapeError,"empty kana at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kana)
+
+        words << Word.new(kanji: kanji,kana: kana)
       end
-      
-      raise ScrapeError,"empty kanji at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kanji)
-      raise ScrapeError,"empty kana at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kana)
-      
-      word = Word.new(kana: kana,kanji: kanji)
-      
-      return word
+
+      return words
     end
-    
+
     # Do not clean and/or strip spaces, as the raw text is important for
-    #   Defn and ArticleScraper.
+    # Defn and ArticleScraper.
     def self.scrape_text_node(tag,url: nil)
       text = tag.text
-      
+
       # No error; empty text is fine (not strictly kanji/kana only).
       return nil if Util.empty_web_str?(text)
-      
+
       word = Word.new(unknown: text)
-      
+
       return word
     end
-    
-    def kanji?()
+
+    def kanji?
       return !Util.empty_web_str?(@kanji)
     end
-    
-    def word()
-      return kanji?() ? @kanji : @kana
+
+    def word
+      return kanji? ? @kanji : @kana
     end
-    
-    def to_s()
-      s = ''.dup()
-      
+
+    def to_s
+      s = ''.dup
+
       s << "'#{@key}': "
       s << "{ kanji=>'#{@kanji}'"
       s << ", kana=>'#{@kana}'"
       s << ", freq=>#{@freq}"
-      s << ", defn=>'#{@defn.to_s().gsub("\n",'\\n')}'"
+      s << ", defn=>'#{@defn.to_s.gsub("\n",'\\n')}'"
       s << ", eng=>'#{@eng}'"
       s << ' }'
-      
+
       return s
     end
   end

data/nhkore.gemspec

@@ -1,42 +1,21 @@
 # encoding: UTF-8
 # frozen_string_literal: true
 
-#--
-# This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
-#++
-
-
-lib = File.expand_path(File.join('..','lib'),__FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-require 'nhkore/version'
-
-Gem::Specification.new() do |spec|
+
+require_relative 'lib/nhkore/version'
+
+Gem::Specification.new do |spec|
   spec.name        = 'nhkore'
   spec.version     = NHKore::VERSION
-  spec.authors     = ['Jonathan Bradley Whited (@esotericpig)']
-  spec.email       = ['bradley@esotericpig.com']
+  spec.authors     = ['Jonathan Bradley Whited']
+  spec.email       = ['code@esotericpig.com']
   spec.licenses    = ['LGPL-3.0-or-later']
   spec.homepage    = 'https://github.com/esotericpig/nhkore'
   spec.summary     = 'NHK News Web (Easy) word frequency (core) scraper for Japanese language learners.'
   spec.description =
     'Scrapes NHK News Web (Easy) for the word frequency (core list) for Japanese language learners.' \
     ' Includes a CLI app and a scraper library.'
-  
+
   spec.metadata = {
     'homepage_uri'      => 'https://github.com/esotericpig/nhkore',
     'source_code_uri'   => 'https://github.com/esotericpig/nhkore',
@@ -46,66 +25,66 @@ Gem::Specification.new() do |spec|
     #'wiki_uri'          => '',
     #'mailing_list_uri'  => '',
   }
-  
+
   spec.requirements = [
     'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html',
   ]
-  
-  spec.required_ruby_version = '>= 2.4'
+
+  spec.required_ruby_version = '>= 2.5'
   spec.require_paths         = ['lib']
   spec.bindir                = 'bin'
   spec.executables           = [spec.name]
-  
+
   spec.files = [
     Dir.glob(File.join("{#{spec.require_paths.join(',')}}",'**','*.{erb,rb}')),
     Dir.glob(File.join(spec.bindir,'*')),
     Dir.glob(File.join('{samples,test,yard}','**','*.{erb,rb}')),
     %W[ Gemfile Gemfile.lock #{spec.name}.gemspec Rakefile .yardopts ],
     %w[ CHANGELOG.md LICENSE.txt README.md ],
-  ].flatten()
-  
+  ].flatten
+
   spec.add_runtime_dependency 'attr_bool'            ,'~> 0.2'  # For attr_accessor?/attr_reader?
   spec.add_runtime_dependency 'bimyou_segmenter'     ,'~> 1.2'  # For splitting Japanese sentences into words
   spec.add_runtime_dependency 'cri'                  ,'~> 2.15' # For CLI commands/options
-  spec.add_runtime_dependency 'down'                 ,'~> 5.1'  # For downloading files (GetCmd)
+  spec.add_runtime_dependency 'down'                 ,'~> 5.2'  # For downloading files (GetCmd)
   spec.add_runtime_dependency 'highline'             ,'~> 2.0'  # For CLI input/output
   spec.add_runtime_dependency 'http-cookie'          ,'~> 1.0'  # For parsing/setting cookies (BingScraper/Scraper)
   spec.add_runtime_dependency 'japanese_deinflector' ,'~> 0.0'  # For unconjugating Japanese words (plain/dictionary form)
-  spec.add_runtime_dependency 'nokogiri'             ,'~> 1.10' # For scraping/hacking
+  spec.add_runtime_dependency 'nokogiri'             ,'~> 1.11' # For scraping/hacking
   spec.add_runtime_dependency 'psychgus'             ,'~> 1.3'  # For styling Psych YAML
   spec.add_runtime_dependency 'public_suffix'        ,'~> 4.0'  # For parsing URL domain names
   spec.add_runtime_dependency 'rainbow'              ,'~> 3.0'  # For CLI color output
   spec.add_runtime_dependency 'rubyzip'              ,'~> 2.3'  # For extracting Zip files (GetCmd)
   spec.add_runtime_dependency 'tiny_segmenter'       ,'~> 0.0'  # For splitting Japanese sentences into words
-  spec.add_runtime_dependency 'tty-progressbar'      ,'~> 0.17' # For CLI progress bars
+  spec.add_runtime_dependency 'tty-progressbar'      ,'~> 0.18' # For CLI progress bars
   spec.add_runtime_dependency 'tty-spinner'          ,'~> 0.9'  # For CLI spinning progress
-  
-  spec.add_development_dependency 'bundler'   ,'~> 2.1'
+
+  spec.add_development_dependency 'bundler'   ,'~> 2.2'
   spec.add_development_dependency 'minitest'  ,'~> 5.14'
   spec.add_development_dependency 'rake'      ,'~> 13.0'
   spec.add_development_dependency 'raketeer'  ,'~> 0.2'  # For extra Rake tasks
-  spec.add_development_dependency 'rdoc'      ,'~> 6.2'  # For YARDoc RDoc (*.rb)
+  spec.add_development_dependency 'rdoc'      ,'~> 6.3'  # For YARDoc RDoc (*.rb)
   spec.add_development_dependency 'redcarpet' ,'~> 3.5'  # For YARDoc Markdown (*.md)
   spec.add_development_dependency 'yard'      ,'~> 0.9'  # For documentation
   spec.add_development_dependency 'yard_ghurt','~> 1.2'  # For extra YARDoc Rake tasks
-  
-  spec.post_install_message = <<-EOM
-  
+
+  spec.post_install_message = <<-MSG
+
   NHKore v#{NHKore::VERSION}
-  
+
   You can now use [#{spec.executables.join(', ')}] on the command line.
-  
+
   Homepage:  #{spec.homepage}
-  
+
   Code:      #{spec.metadata['source_code_uri']}
   Bugs:      #{spec.metadata['bug_tracker_uri']}
-  
+
   Changelog: #{spec.metadata['changelog_uri']}
-  
-  EOM
-  
+
+  MSG
+
   spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
-  
+
   spec.rdoc_options = [
     '--hyperlink-all','--show-hash',
     '--title',"NHKore v#{NHKore::VERSION} Doc",