nhkore 0.3.14 → 0.3.16

data/lib/nhkore/error.rb

@@ -10,21 +10,11 @@
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Error < ::StandardError; end
 
-  # @since 0.2.0
   class CLIError < Error; end
-
-  # @since 0.2.0
+  class Http404Error < Error; end
   class ParseError < Error; end
-
-  # @since 0.2.0
   class ScrapeError < Error; end
-
-  # @since 0.2.0
   class ZipError < Error; end
 end

data/lib/nhkore/fileable.rb

@@ -10,10 +10,6 @@
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module Fileable
     def self.included(mod)
       mod.extend ClassMethods

data/lib/nhkore/lib.rb

@@ -38,9 +38,6 @@ module NHKore
   ###
   # Include this file to only require the files needed to use this
   # Gem as a library (i.e., don't include CLI-related files).
-  #
-  # @author Jonathan Bradley Whited
-  # @since  0.3.2
   ###
   module Lib
   end

data/lib/nhkore/missingno.rb

@@ -13,10 +13,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Missingno
     attr_reader :kanas
     attr_reader :kanjis
@@ -68,13 +64,13 @@ module NHKore
     def kana_from_kanji(kanji)
       word = @kanjis[kanji]
 
-      return word.nil? ? nil : word.kana
+      return word&.kana
     end
 
     def kanji_from_kana(kana)
       word = @kanas[kana]
 
-      return word.nil? ? nil : word.kanji
+      return word&.kanji
     end
   end
 end

data/lib/nhkore/news.rb

@@ -16,10 +16,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class News
     include Fileable
 
@@ -30,7 +26,7 @@ module NHKore
     attr_reader :sha256s
 
     def initialize
-      super()
+      super
 
       @articles = {}
       @sha256s = {}
@@ -127,10 +123,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class FutsuuNews < News
     DEFAULT_FILENAME = 'nhk_news_web_regular.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
@@ -144,14 +136,10 @@ module NHKore
     end
 
     def save_file(file=DEFAULT_FILE,**kargs)
-      super(file,**kargs)
+      super
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class YasashiiNews < News
     DEFAULT_FILENAME = 'nhk_news_web_easy.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
@@ -165,7 +153,7 @@ module NHKore
     end
 
     def save_file(file=DEFAULT_FILE,**kargs)
-      super(file,**kargs)
+      super
     end
   end
 end

data/lib/nhkore/polisher.rb

@@ -13,10 +13,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Polisher
     def begin_polish(str)
       return str
@@ -52,10 +48,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicPolisher < Polisher
     def end_polish(str)
       # Keep Japanese dots in names:
@@ -72,10 +64,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestPolisher < BasicPolisher
   end
 end

data/lib/nhkore/scraper.rb

@@ -13,15 +13,12 @@ require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
 
+require 'nhkore/error'
 require 'nhkore/user_agents'
 require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Scraper
     extend AttrBool::Ext
 
@@ -177,7 +174,13 @@ module NHKore
         retry
       # Must come after HTTPRedirect since a subclass of HTTPError.
       rescue OpenURI::HTTPError => e
-        raise e.exception("HTTP error[#{e}] at URL[#{url}]")
+        msg = "HTTP error[#{e}] at URL[#{url}]"
+
+        if e.to_s.include?('404 Not Found')
+          raise Http404Error,msg
+        else
+          raise e.exception(msg)
+        end
       rescue SocketError => e
         if (max_retries -= 1) < 0
           raise e.exception("Socket error[#{e}] at URL[#{url}]")

data/lib/nhkore/search_link.rb

@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchLink
     extend AttrBool::Ext
 
@@ -45,11 +41,11 @@ module NHKore
     def encode_with(coder)
       # Order matters.
 
-      coder[:url] = @url.nil? ? nil : @url.to_s
+      coder[:url] = @url&.to_s
       coder[:scraped] = @scraped
-      coder[:datetime] = @datetime.nil? ? nil : @datetime.iso8601
+      coder[:datetime] = @datetime&.iso8601
       coder[:title] = @title
-      coder[:futsuurl] = @futsuurl.nil? ? nil : @futsuurl.to_s
+      coder[:futsuurl] = @futsuurl&.to_s
       coder[:sha256] = @sha256
     end
 
@@ -86,13 +82,13 @@ module NHKore
     end
 
     def futsuurl=(value)
-      # Don't store URI, store String.
-      @futsuurl = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @futsuurl = value&.to_s
     end
 
     def url=(value)
-      # Don't store URI, store String.
-      @url = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @url = value&.to_s
     end
 
     def to_s(mini: false)
@@ -114,10 +110,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchLinks
     include Fileable
 
@@ -136,13 +128,13 @@ module NHKore
     attr_reader :links
 
     def initialize
-      super()
+      super
 
       @links = {}
     end
 
     def add_link(link)
-      url = link.url.nil? ? nil : link.url.to_s
+      url = link.url&.to_s
 
       return self if @links.key?(url)
 

data/lib/nhkore/search_scraper.rb

@@ -9,6 +9,7 @@
 #++
 
 
+require 'net/http'
 require 'uri'
 
 require 'nhkore/error'
@@ -18,10 +19,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchScraper < Scraper
     DEFAULT_RESULT_COUNT = 100
     FUTSUU_SITE = 'nhk.or.jp/news/html/'
@@ -34,10 +31,11 @@ module NHKore
     YASASHII_REGEX = /\A[^.]+\.#{Regexp.quote(YASASHII_SITE)}.+\.html?/i.freeze
 
     IGNORE_LINK_REGEX = %r{
-      /about\.html?             # https://www3.nhk.or.jp/news/easy/about.html
-      |/movieplayer\.html?      # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
-      |/audio\.html?            # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
-      |/news/easy/index\.html?  # http://www3.nhk.or.jp/news/easy/index.html
+      /about\.html?               # https://www3.nhk.or.jp/news/easy/about.html
+      |/movieplayer\.html?        # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
+      |/audio\.html?              # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
+      |/news/easy/index\.html?    # https://www3.nhk.or.jp/news/easy/index.html
+      |/disaster_earthquake.html  # https://www3.nhk.or.jp/news/easy/article/disaster_earthquake.html
 
       # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
       # https://www3.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10012689671000&title=「鬼滅の刃」の映画が台湾でも始まって大勢の人が見に行く
@@ -47,7 +45,7 @@ module NHKore
     # Search Engines are strict, so trigger using the default HTTP header fields
     # with +header: {}+ and fetch/set the cookie using +eat_cookie: true+.
     def initialize(url,eat_cookie: true,header: {},**kargs)
-      super(url,eat_cookie: eat_cookie,header: header,**kargs)
+      super
     end
 
     def ignore_link?(link,cleaned: true)
@@ -56,17 +54,35 @@ module NHKore
       link = Util.unspace_web_str(link).downcase unless cleaned
 
       return true if link.empty?
-
       return true if IGNORE_LINK_REGEX.match?(link)
-
       return false
     end
+
+    # Example: https://www3.nhk.or.jp/news/easy/k10014150691000/k10014150691000.html
+    def fetch_valid_link?(link)
+      uri = begin
+        URI(link)
+      rescue StandardError
+        return false # Bad URL.
+      end
+
+      begin
+        ssl = uri.scheme.to_s.strip.downcase.include?('https')
+
+        Net::HTTP.start(uri.host,uri.port,use_ssl: ssl) do |http|
+          resp = http.head(uri.request_uri)
+          code = resp.code
+
+          return code != '404'
+        end
+      rescue StandardError
+        # Ignore; try actually scraping the article anyway.
+      end
+
+      return true
+    end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BingScraper < SearchScraper
     attr_reader :regex
     attr_reader :site
@@ -136,9 +152,8 @@ module NHKore
             next_page.count = count
             next_page.url = join_url(href)
           end
-        elsif href =~ regex
+        elsif href =~ regex && fetch_valid_link?(href)
           slinks.add_link(SearchLink.new(href))
-
           link_count += 1
         end
       end
@@ -165,10 +180,9 @@ module NHKore
           rss_links << link
 
           next if ignore_link?(link)
-          next if link !~ regex
+          next if link !~ regex || !fetch_valid_link?(link)
 
           slinks.add_link(SearchLink.new(link))
-
           link_count += 1
         end
 
@@ -192,17 +206,13 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class NextPage
     attr_accessor :count
     attr_accessor :rss_links
     attr_accessor :url
 
     def initialize
-      super()
+      super
 
       @count = -1
       @rss_links = nil

data/lib/nhkore/sifter.rb

@@ -15,10 +15,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Sifter
     include Fileable
 
@@ -61,10 +57,8 @@ module NHKore
     end
 
     def build_rows(words)
-      rows = []
-
-      words.each do |word|
-        rows << build_word_row(word)
+      rows = words.map do |word|
+        build_word_row(word)
       end
 
       return rows
@@ -336,6 +330,11 @@ module NHKore
         next if filter?(article)
 
         article.words.each_value do |word|
+          # TODO: Try to remove garbage data better.
+          next if word.word.length < 2
+          next if word.freq <= 1
+          next if word.word =~ /\p{Latin}|[[:digit:]]/
+
           master_article.add_word(word,use_freq: true)
         end
       end

data/lib/nhkore/splitter.rb

@@ -13,10 +13,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Splitter
     def begin_split(str)
       return str
@@ -30,19 +26,12 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicSplitter < Splitter
     def end_split(str)
       return str.split(Util::NORMALIZE_STR_REGEX)
     end
   end
 
-  ###
-  # @since  0.2.0
-  ###
   class BimyouSplitter < Splitter
     def initialize(*)
       require 'bimyou_segmenter'
@@ -55,9 +44,6 @@ module NHKore
     end
   end
 
-  ###
-  # @since  0.2.0
-  ###
   class TinySplitter < Splitter
     attr_accessor :tiny
 
@@ -74,10 +60,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestSplitter < BimyouSplitter
   end
 end

data/lib/nhkore/user_agents.rb

@@ -38,9 +38,6 @@ module NHKore
   #
   # The gem is really old and had a lot of warnings, so decided to make this class.
   # Maybe I'll fork the gem and maintain a new version in the future...
-  #
-  # @author Jonathan Bradley Whited
-  # @since  0.2.1
   ###
   class UserAgents
     attr_accessor :data
@@ -53,7 +50,7 @@ module NHKore
     # because we don't need all of the data in memory after getting just 1
     # sample, even though it's slower.
     def initialize
-      super()
+      super
 
       # rubocop:disable all
       @data = [

data/lib/nhkore/util.rb

@@ -16,10 +16,6 @@ require 'uri'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module Util
     CORE_DIR = 'core'
     WEB_DIR = 'web'

data/lib/nhkore/variator.rb

@@ -10,10 +10,6 @@
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Variator
     def begin_variate(str)
       return str
@@ -27,10 +23,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicVariator < Variator
     def end_variate(str)
       return [] # No variations; don't return nil
@@ -41,8 +33,6 @@ module NHKore
   # Guesses a word's dictionary/plain form (辞書形).
   #
   # It doesn't work very well,but better than nothing...
-  #
-  # @since  0.2.0
   ###
   class DictFormVariator < Variator
     attr_accessor :deinflector
@@ -66,10 +56,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestVariator < DictFormVariator
   end
 end

data/lib/nhkore/version.rb

@@ -10,5 +10,5 @@
 
 
 module NHKore
-  VERSION = '0.3.14'
+  VERSION = '0.3.16'
 end

data/lib/nhkore/word.rb

@@ -16,10 +16,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.1.0
-  ###
   class Word
     attr_accessor :defn
     attr_accessor :eng

data/lib/nhkore.rb

@@ -26,12 +26,7 @@ require 'nhkore/cli/search_cmd'
 require 'nhkore/cli/sift_cmd'
 
 
-###
-# @author Jonathan Bradley Whited
-# @since  0.1.0
-###
 module NHKore
-  # @since 0.2.0
   def self.run(args=ARGV)
     app = App.new(args)
 

data/nhkore.gemspec

@@ -54,27 +54,27 @@ Gem::Specification.new do |spec|
   run_dep[ 'attr_bool'           ,'~> 0.2'  ] # attr_accessor?/attr_reader?.
   run_dep[ 'bimyou_segmenter'    ,'~> 1.2'  ] # Splitting Japanese sentences into words.
   run_dep[ 'cri'                 ,'~> 2.15' ] # CLI commands/options.
-  run_dep[ 'down'                ,'~> 5.3'  ] # Downloading files (GetCmd).
-  run_dep[ 'highline'            ,'~> 2.0'  ] # CLI input/output.
+  run_dep[ 'down'                ,'~> 5.4'  ] # Downloading files (GetCmd).
+  run_dep[ 'highline'            ,'~> 3.1'  ] # CLI input/output.
   run_dep[ 'http-cookie'         ,'~> 1.0'  ] # Parsing/Setting cookies [(Bing)Scraper].
   run_dep[ 'japanese_deinflector','~> 0.0'  ] # Unconjugating Japanese words (dictionary form).
-  run_dep[ 'nokogiri'            ,'~> 1.13' ] # Scraping/Hacking.
+  run_dep[ 'nokogiri'            ,'~> 1.16' ] # Scraping/Hacking.
   run_dep[ 'psychgus'            ,'~> 1.3'  ] # Styling Psych YAML.
-  run_dep[ 'public_suffix'       ,'~> 4.0'  ] # Parsing URL domain names.
+  run_dep[ 'public_suffix'       ,'~> 6.0'  ] # Parsing URL domain names.
   run_dep[ 'rainbow'             ,'~> 3.1'  ] # CLI color output.
-  run_dep[ 'rss'                 ,'~> 0.2'  ] # Scraping [(Bing)Scraper].
+  run_dep[ 'rss'                 ,'~> 0.3'  ] # Scraping [(Bing)Scraper].
   run_dep[ 'rubyzip'             ,'~> 2.3'  ] # Extracting Zip files (GetCmd).
   run_dep[ 'tiny_segmenter'      ,'~> 0.0'  ] # Splitting Japanese sentences into words.
   run_dep[ 'tty-progressbar'     ,'~> 0.18' ] # CLI progress bars.
   run_dep[ 'tty-spinner'         ,'~> 0.9'  ] # CLI spinning progress.
 
   dev_dep = spec.method(:add_development_dependency)
-  dev_dep[ 'bundler'   ,'~> 2.3'  ]
-  dev_dep[ 'minitest'  ,'~> 5.16' ]
-  dev_dep[ 'rake'      ,'~> 13.0' ]
+  dev_dep[ 'bundler'   ,'~> 2.5'  ]
+  dev_dep[ 'minitest'  ,'~> 5.25' ]
+  dev_dep[ 'rake'      ,'~> 13.2' ]
   dev_dep[ 'raketeer'  ,'~> 0.2'  ] # Extra Rake tasks.
-  dev_dep[ 'rdoc'      ,'~> 6.4'  ] # YARDoc RDoc (*.rb).
-  dev_dep[ 'redcarpet' ,'~> 3.5'  ] # YARDoc Markdown (*.md).
+  dev_dep[ 'rdoc'      ,'~> 6.7'  ] # YARDoc RDoc (*.rb).
+  dev_dep[ 'redcarpet' ,'~> 3.6'  ] # YARDoc Markdown (*.md).
   dev_dep[ 'yard'      ,'~> 0.9'  ] # Doc.
   dev_dep[ 'yard_ghurt','~> 1.2'  ] # Extra YARDoc Rake tasks.
 

data/samples/looper.rb

@@ -13,9 +13,6 @@
 ###
 # If you run this script, be aware that it uses the +-F+ force option
 # (which overwrites files without prompting).
-#
-# @author Jonathan Bradley Whited
-# @since  0.3.2
 ###
 
 case ARGV[0]