nhkore 0.3.13 → 0.3.16

data/lib/nhkore/cli/search_cmd.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.3.0
-  ###
   module SearchCmd
     def build_search_cmd
       app = self
@@ -42,6 +38,12 @@ module CLI
         DESC
           app.check_empty_opt(:in,value)
         }
+        option :l,:loop,'number of times to repeat the search to ensure results',argument: :required,
+          transform: lambda { |value|
+            value = value.to_i
+            value = 1 if value < 1
+            value
+          }
         option :o,:out,<<-DESC,argument: :required,transform: lambda { |value|
           'directory/file' to save links to; if you only specify a directory or a file, it will attach the
           appropriate default directory/file name
@@ -164,6 +166,8 @@ module CLI
 
       dry_run = @cmd_opts[:dry_run]
       in_file = @cmd_opts[:in]
+      loop_times = @cmd_opts[:loop]
+      loop_times = 1 if loop_times.nil? || loop_times < 1
       out_file = @cmd_opts[:out]
       result_count = @cmd_opts[:results]
       result_count = SearchScraper::DEFAULT_RESULT_COUNT if result_count.nil?
@@ -174,9 +178,6 @@ module CLI
       is_file = !in_file.nil?
       links = nil
       new_links = [] # For --dry-run
-      next_page = NextPage.new
-      page_count = 0
-      page_num = 1
       url = in_file # nil will use default URL, else a file
 
       # Load previous links for 'scraped?' vars.
@@ -196,43 +197,52 @@ module CLI
         end
 
         puts "#{scraped_count} of #{links_count} links scraped."
-
         return
       end
 
-      range = (0..10_000) # Do a range to prevent an infinite loop; ichiman!
-
-      case search_type
-      # Anything that extends SearchScraper.
-      when :bing
-        range.each do
-          scraper = nil
-
-          case search_type
-          when :bing
-            scraper = BingScraper.new(nhk_type,count: result_count,is_file: is_file,url: url,**@scraper_kargs)
-          else
-            raise NHKore::Error,"internal code broken; add missing search_type[#{search_type}]"
-          end
+      1.upto(loop_times) do |loop_i|
+        page_range = (0..10_000) # Do a range to prevent an infinite loop; ichiman!
+
+        next_page = NextPage.new
+        page_count = 0
+        page_num = 1
+
+        case search_type
+        # Anything that extends SearchScraper.
+        when :bing
+          page_range.each do
+            scraper = nil
+
+            case search_type
+            when :bing
+              scraper = BingScraper.new(
+                nhk_type,count: result_count,is_file: is_file,url: url,**@scraper_kargs
+              )
+            else
+              raise NHKore::Error,"internal code broken; add missing search_type[#{search_type}]"
+            end
 
-          next_page = scraper.scrape(links,next_page)
+            next_page = scraper.scrape(links,next_page)
 
-          new_links.concat(links.links.values[links_count..-1])
-          links_count = links.length
-          page_count = next_page.count if next_page.count > 0
+            new_links.concat(links.links.values[links_count..])
+            links_count = links.length
+            page_count = next_page.count if next_page.count > 0
 
-          update_spin_detail(" (page=#{page_num}, count=#{page_count}, links=#{links.length}," \
-            " new_links=#{new_links.length})")
+            update_spin_detail(
+              format(' (%d/%d, page=%d, count=%d, links=%d, new_links=%d)',
+                     loop_i,loop_times,page_num,page_count,links.length,new_links.length)
+            )
 
-          break if next_page.empty?
+            break if next_page.empty?
 
-          page_num += 1
-          url = next_page.url
+            page_num += 1
+            url = next_page.url
 
-          sleep_scraper
+            sleep_scraper
+          end
+        else
+          raise ArgumentError,"invalid search_type[#{search_type}]"
         end
-      else
-        raise ArgumentError,"invalid search_type[#{search_type}]"
       end
 
       stop_spin

data/lib/nhkore/cli/sift_cmd.rb

@@ -20,10 +20,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module SiftCmd
     DEFAULT_SIFT_EXT = :csv
     DEFAULT_SIFT_FUTSUU_FILE = "#{Sifter::DEFAULT_FUTSUU_FILE}{search.criteria}{file.ext}"
@@ -260,7 +256,7 @@ module CLI
       puts
 
       if dry_run
-        puts sifter.to_s
+        puts sifter
       else
         start_spin('Saving sifted data to file')
 

data/lib/nhkore/datetime_parser.rb

@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.3.4
-  ###
   class DatetimeParser
     extend AttrBool::Ext
 
@@ -181,7 +177,7 @@ module NHKore
       return self if @min_or_max
 
       has_small = false
-      jst_now = Util.jst_now()
+      jst_now = Util.jst_now
 
       # Must be from smallest to biggest.
 

data/lib/nhkore/defn.rb

@@ -16,17 +16,13 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Defn
     attr_reader :hyoukis
     attr_accessor :text
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @hyoukis = []
       @text = ''.dup

data/lib/nhkore/dict.rb

@@ -14,15 +14,11 @@ require 'nhkore/error'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Dict
     attr_reader :entries
 
     def initialize
-      super()
+      super
 
       @entries = {}
     end
@@ -39,6 +35,7 @@ module NHKore
       dict = Dict.new
 
       hash.each do |id,array|
+        id = id.to_s.strip.downcase # 'RSHOK-K-003806', '0000'
         entry = Entry.scrape(id,array,missingno: missingno,url: url)
 
         next if entry.nil?

data/lib/nhkore/dict_scraper.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -16,10 +16,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class DictScraper < Scraper
     attr_accessor :missingno
 
@@ -39,7 +35,7 @@ module NHKore
       i = url.rindex(%r{[/\\]}) # Can be a URL or a file
       i = i.nil? ? 0 : (i + 1) # If no match found, no path
 
-      basename = File.basename(url[i..-1],'.*') if basename.nil?
+      basename = File.basename(url[i..],'.*') if basename.nil?
       path = url[0...i]
 
       return "#{path}#{basename}.out.dic"

data/lib/nhkore/entry.rb

@@ -14,10 +14,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Entry
     HYOUKI_SEP = '・'
 
@@ -25,18 +21,16 @@ module NHKore
     attr_accessor :id
 
     def initialize
-      super()
+      super
 
       @defns = []
       @id = nil
     end
 
     def build_defn
-      defns = []
       i = 0
-
-      @defns.each do |defn|
-        defns << "#{i += 1}）#{defn}" # Japanese parenthesis
+      defns = @defns.map do |defn|
+        "#{i += 1}）#{defn}" # Japanese parenthesis
       end
 
       return defns.join("\n")

data/lib/nhkore/error.rb

@@ -10,21 +10,11 @@
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Error < ::StandardError; end
 
-  # @since 0.2.0
   class CLIError < Error; end
-
-  # @since 0.2.0
+  class Http404Error < Error; end
   class ParseError < Error; end
-
-  # @since 0.2.0
   class ScrapeError < Error; end
-
-  # @since 0.2.0
   class ZipError < Error; end
 end

data/lib/nhkore/fileable.rb

@@ -10,10 +10,6 @@
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module Fileable
     def self.included(mod)
       mod.extend ClassMethods

data/lib/nhkore/lib.rb

@@ -38,9 +38,6 @@ module NHKore
   ###
   # Include this file to only require the files needed to use this
   # Gem as a library (i.e., don't include CLI-related files).
-  #
-  # @author Jonathan Bradley Whited
-  # @since  0.3.2
   ###
   module Lib
   end

data/lib/nhkore/missingno.rb

@@ -13,10 +13,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Missingno
     attr_reader :kanas
     attr_reader :kanjis
@@ -68,13 +64,13 @@ module NHKore
     def kana_from_kanji(kanji)
       word = @kanjis[kanji]
 
-      return word.nil? ? nil : word.kana
+      return word&.kana
     end
 
     def kanji_from_kana(kana)
       word = @kanas[kana]
 
-      return word.nil? ? nil : word.kanji
+      return word&.kanji
     end
   end
 end

data/lib/nhkore/news.rb

@@ -16,10 +16,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class News
     include Fileable
 
@@ -30,7 +26,7 @@ module NHKore
     attr_reader :sha256s
 
     def initialize
-      super()
+      super
 
       @articles = {}
       @sha256s = {}
@@ -127,10 +123,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class FutsuuNews < News
     DEFAULT_FILENAME = 'nhk_news_web_regular.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
@@ -144,14 +136,10 @@ module NHKore
     end
 
     def save_file(file=DEFAULT_FILE,**kargs)
-      super(file,**kargs)
+      super
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class YasashiiNews < News
     DEFAULT_FILENAME = 'nhk_news_web_easy.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
@@ -165,7 +153,7 @@ module NHKore
     end
 
     def save_file(file=DEFAULT_FILE,**kargs)
-      super(file,**kargs)
+      super
     end
   end
 end

data/lib/nhkore/polisher.rb

@@ -13,10 +13,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Polisher
     def begin_polish(str)
       return str
@@ -52,10 +48,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicPolisher < Polisher
     def end_polish(str)
       # Keep Japanese dots in names:
@@ -72,10 +64,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestPolisher < BasicPolisher
   end
 end

data/lib/nhkore/scraper.rb

@@ -13,15 +13,12 @@ require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
 
+require 'nhkore/error'
 require 'nhkore/user_agents'
 require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Scraper
     extend AttrBool::Ext
 
@@ -177,7 +174,13 @@ module NHKore
         retry
       # Must come after HTTPRedirect since a subclass of HTTPError.
       rescue OpenURI::HTTPError => e
-        raise e.exception("HTTP error[#{e}] at URL[#{url}]")
+        msg = "HTTP error[#{e}] at URL[#{url}]"
+
+        if e.to_s.include?('404 Not Found')
+          raise Http404Error,msg
+        else
+          raise e.exception(msg)
+        end
       rescue SocketError => e
         if (max_retries -= 1) < 0
           raise e.exception("Socket error[#{e}] at URL[#{url}]")

data/lib/nhkore/search_link.rb

@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchLink
     extend AttrBool::Ext
 
@@ -45,11 +41,11 @@ module NHKore
     def encode_with(coder)
       # Order matters.
 
-      coder[:url] = @url.nil? ? nil : @url.to_s
+      coder[:url] = @url&.to_s
       coder[:scraped] = @scraped
-      coder[:datetime] = @datetime.nil? ? nil : @datetime.iso8601
+      coder[:datetime] = @datetime&.iso8601
       coder[:title] = @title
-      coder[:futsuurl] = @futsuurl.nil? ? nil : @futsuurl.to_s
+      coder[:futsuurl] = @futsuurl&.to_s
       coder[:sha256] = @sha256
     end
 
@@ -86,13 +82,13 @@ module NHKore
     end
 
     def futsuurl=(value)
-      # Don't store URI, store String.
-      @futsuurl = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @futsuurl = value&.to_s
     end
 
     def url=(value)
-      # Don't store URI, store String.
-      @url = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @url = value&.to_s
     end
 
     def to_s(mini: false)
@@ -114,10 +110,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchLinks
     include Fileable
 
@@ -136,13 +128,13 @@ module NHKore
     attr_reader :links
 
     def initialize
-      super()
+      super
 
       @links = {}
     end
 
     def add_link(link)
-      url = link.url.nil? ? nil : link.url.to_s
+      url = link.url&.to_s
 
       return self if @links.key?(url)
 

data/lib/nhkore/search_scraper.rb

@@ -9,6 +9,7 @@
 #++
 
 
+require 'net/http'
 require 'uri'
 
 require 'nhkore/error'
@@ -18,10 +19,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class SearchScraper < Scraper
     DEFAULT_RESULT_COUNT = 100
     FUTSUU_SITE = 'nhk.or.jp/news/html/'
@@ -34,10 +31,11 @@ module NHKore
     YASASHII_REGEX = /\A[^.]+\.#{Regexp.quote(YASASHII_SITE)}.+\.html?/i.freeze
 
     IGNORE_LINK_REGEX = %r{
-      /about\.html?             # https://www3.nhk.or.jp/news/easy/about.html
-      |/movieplayer\.html?      # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
-      |/audio\.html?            # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
-      |/news/easy/index\.html?  # http://www3.nhk.or.jp/news/easy/index.html
+      /about\.html?               # https://www3.nhk.or.jp/news/easy/about.html
+      |/movieplayer\.html?        # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
+      |/audio\.html?              # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
+      |/news/easy/index\.html?    # https://www3.nhk.or.jp/news/easy/index.html
+      |/disaster_earthquake.html  # https://www3.nhk.or.jp/news/easy/article/disaster_earthquake.html
 
       # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
       # https://www3.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10012689671000&title=「鬼滅の刃」の映画が台湾でも始まって大勢の人が見に行く
@@ -47,7 +45,7 @@ module NHKore
     # Search Engines are strict, so trigger using the default HTTP header fields
     # with +header: {}+ and fetch/set the cookie using +eat_cookie: true+.
     def initialize(url,eat_cookie: true,header: {},**kargs)
-      super(url,eat_cookie: eat_cookie,header: header,**kargs)
+      super
     end
 
     def ignore_link?(link,cleaned: true)
@@ -56,17 +54,35 @@ module NHKore
       link = Util.unspace_web_str(link).downcase unless cleaned
 
       return true if link.empty?
-
       return true if IGNORE_LINK_REGEX.match?(link)
-
       return false
     end
+
+    # Example: https://www3.nhk.or.jp/news/easy/k10014150691000/k10014150691000.html
+    def fetch_valid_link?(link)
+      uri = begin
+        URI(link)
+      rescue StandardError
+        return false # Bad URL.
+      end
+
+      begin
+        ssl = uri.scheme.to_s.strip.downcase.include?('https')
+
+        Net::HTTP.start(uri.host,uri.port,use_ssl: ssl) do |http|
+          resp = http.head(uri.request_uri)
+          code = resp.code
+
+          return code != '404'
+        end
+      rescue StandardError
+        # Ignore; try actually scraping the article anyway.
+      end
+
+      return true
+    end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BingScraper < SearchScraper
     attr_reader :regex
     attr_reader :site
@@ -136,9 +152,8 @@ module NHKore
             next_page.count = count
             next_page.url = join_url(href)
           end
-        elsif href =~ regex
+        elsif href =~ regex && fetch_valid_link?(href)
           slinks.add_link(SearchLink.new(href))
-
           link_count += 1
         end
       end
@@ -165,10 +180,9 @@ module NHKore
           rss_links << link
 
           next if ignore_link?(link)
-          next if link !~ regex
+          next if link !~ regex || !fetch_valid_link?(link)
 
           slinks.add_link(SearchLink.new(link))
-
           link_count += 1
         end
 
@@ -192,17 +206,13 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class NextPage
     attr_accessor :count
     attr_accessor :rss_links
     attr_accessor :url
 
     def initialize
-      super()
+      super
 
       @count = -1
       @rss_links = nil