nhkore 0.3.17 → 0.3.19

data/lib/nhkore/article_scraper.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'attr_bool'
 require 'digest'
 
@@ -24,7 +23,6 @@ require 'nhkore/util'
 require 'nhkore/variator'
 require 'nhkore/word'
 
-
 module NHKore
   class ArticleScraper < Scraper
     extend AttrBool::Ext
@@ -47,8 +45,8 @@ module NHKore
     #                  instead of raising an error
     # @param strict [true,false]
     def initialize(url,cleaners: [BestCleaner.new],datetime: nil,dict: :scrape,missingno: nil,
-        polishers: [BestPolisher.new],splitter: BestSplitter.new,strict: true,
-        variators: [BestVariator.new],year: nil,**kargs)
+                   polishers: [BestPolisher.new],splitter: BestSplitter.new,strict: true,
+                   variators: [BestVariator.new],year: nil,**kargs)
       super(url,**kargs)
 
       @cleaners = Array(cleaners)
@@ -179,13 +177,13 @@ module NHKore
 
     def scrape_content(doc,article)
       tag = doc.css('div#js-article-body')
-      tag = doc.css('div.article-main__body') if tag.length < 1
-      tag = doc.css('div.article-body') if tag.length < 1
+      tag = doc.css('div.article-main__body') if tag.empty?
+      tag = doc.css('div.article-body') if tag.empty?
 
       # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
-      tag = doc.css('div#main') if tag.length < 1 && !@strict
+      tag = doc.css('div#main') if tag.empty? && !@strict
 
-      if tag.length > 0
+      if !tag.empty?
         text = Util.unspace_web_str(tag.text.to_s)
 
         if !text.empty?
@@ -202,14 +200,14 @@ module NHKore
       raise ScrapeError,"could not scrape content at URL[#{@url}]"
     end
 
-    def scrape_datetime(doc,futsuurl=nil)
+    def scrape_datetime(doc,futsuurl = nil)
       year = scrape_year(doc,futsuurl)
 
       # First, try with the id.
       tag_name = 'p#js-article-date'
       tag = doc.css(tag_name)
 
-      if tag.length > 0
+      if !tag.empty?
         tag_text = tag[0].text
 
         begin
@@ -226,7 +224,7 @@ module NHKore
       tag_name = 'p.article-main__date'
       tag = doc.css(tag_name)
 
-      if tag.length > 0
+      if !tag.empty?
         tag_text = tag[0].text
 
         begin
@@ -244,10 +242,10 @@ module NHKore
       # - 'news20170331_k10010922481000'
       tag = doc.css('body')
 
-      if tag.length > 0
+      if !tag.empty?
         tag_id = tag[0]['id'].to_s.split('_',2)
 
-        if tag_id.length > 0
+        if !tag_id.empty?
           tag_id = tag_id[0].gsub(/[^[[:digit:]]]+/,'')
 
           if tag_id.length == 8
@@ -272,8 +270,8 @@ module NHKore
 
       begin
         scraper = DictScraper.new(dict_url,missingno: @missingno,parse_url: false,**@kargs)
-      rescue OpenURI::HTTPError => e
-        if retries == 0 && e.to_s.include?('404')
+      rescue Http404Error => e
+        if retries == 0
           read
 
           scraper = ArticleScraper.new(@url,str_or_io: @str_or_io,**@kargs)
@@ -283,7 +281,10 @@ module NHKore
 
           retry
         else
-          raise e.exception("could not scrape dictionary URL[#{dict_url}] at URL[#{@url}]: #{e}")
+          # raise e.exception("failed to scrape dictionary URL[#{dict_url}] at URL[#{@url}]: #{e}")
+          Util.warn("failed to scrape dictionary URL[#{dict_url}] at URL[#{@url}]: #{e}")
+          @dict = nil
+          return
         end
       end
 
@@ -297,7 +298,7 @@ module NHKore
       # - 'news20170331_k10010922481000'
       tag = doc.css('body')
 
-      if tag.length > 0
+      if !tag.empty?
         tag_id = tag[0]['id'].to_s.split('_',2)
 
         if tag_id.length == 2
@@ -360,7 +361,7 @@ module NHKore
       # First, try with the id.
       tag = doc.css('div#js-regular-news-wrapper')
 
-      if tag.length > 0
+      if !tag.empty?
         link = scrape_link(tag[0])
 
         return link unless link.nil?
@@ -369,7 +370,7 @@ module NHKore
       # Second, try with the class.
       tag = doc.css('div.link-to-normal')
 
-      if tag.length > 0
+      if !tag.empty?
         link = scrape_link(tag[0])
 
         return link unless link.nil?
@@ -385,7 +386,7 @@ module NHKore
     def scrape_link(tag)
       link = tag.css('a')
 
-      return nil if link.length < 1
+      return nil if link.empty?
 
       link = Util.unspace_web_str(link[0]['href'].to_s)
 
@@ -493,24 +494,24 @@ module NHKore
       tag = doc.css('h1.article-main__title')
       tag_name = nil
 
-      if tag.length < 1
+      if tag.empty?
         # - https://www3.nhk.or.jp/news/easy/em2024081312029/em2024081312029.html
         tag = doc.css('h1.article-title') # No warning.
       end
 
-      if tag.length < 1
+      if tag.empty?
         # - https://www3.nhk.or.jp/news/easy/article/disaster_earthquake_illust.html
         tag_name = 'h1.article-eq__title'
         tag = doc.css(tag_name)
       end
-      if tag.length < 1 && !@strict
+      if tag.empty? && !@strict
         # This shouldn't be used except for select sites.
         # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
         tag_name = 'div#main h2'
         tag = doc.css(tag_name)
       end
 
-      if tag.length > 0
+      if !tag.empty?
         Util.warn("using [#{tag_name}] for title at URL[#{@url}]") unless tag_name.nil?
 
         result = scrape_and_add_words(tag,article)
@@ -548,8 +549,8 @@ module NHKore
 
             if klass == 'dicwin' && !id.nil?
               if dicwin
-                raise ScrapeError,"invalid dicWin class[#{child}] nested inside another dicWin class at" \
-                  " URL[#{@url}]"
+                raise ScrapeError,"invalid dicWin class[#{child}] nested inside another dicWin class at " \
+                                  "URL[#{@url}]"
               end
 
               dicwin_id = id
@@ -582,11 +583,11 @@ module NHKore
       return result
     end
 
-    def scrape_year(doc,futsuurl=nil)
+    def scrape_year(doc,futsuurl = nil)
       # First, try body's id.
       tag = doc.css('body')
 
-      if tag.length > 0
+      if !tag.empty?
         tag_id = tag[0]['id'].to_s.gsub(/[^[[:digit:]]]+/,'')
 
         if tag_id.length >= 4

data/lib/nhkore/cleaner.rb

@@ -3,16 +3,14 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/util'
 require 'nhkore/word'
 
-
 module NHKore
   class Cleaner
     def begin_clean(str)

data/lib/nhkore/cli/fx_cmd.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 module NHKore
 module CLI
   module FXCmd
@@ -40,13 +39,11 @@ module CLI
     end
 
     def test_fx_progress_bar
-      bars = nil
-
-      if @cmd_opts[:all]
-        bars = {default: :default,classic: :classic,no: :no}
-      else
-        bars = {user: @progress_bar}
-      end
+      bars = if @cmd_opts[:all]
+               {default: :default,classic: :classic,no: :no}
+             else
+               {user: @progress_bar}
+             end
 
       bars.each do |name,bar|
         name = name.to_s.capitalize
@@ -65,19 +62,17 @@ module CLI
 
     def test_fx_spinner
       app_spinner = @spinner
-      spinners = nil
-
-      if @cmd_opts[:all]
-        spinners = {
-          default: App::DEFAULT_SPINNER,
-          classic: App::CLASSIC_SPINNER,
-          no: {},
-        }
-      else
-        spinners = {
-          user: app_spinner
-        }
-      end
+      spinners = if @cmd_opts[:all]
+                   {
+                     default: App::DEFAULT_SPINNER,
+                     classic: App::CLASSIC_SPINNER,
+                     no: {},
+                   }
+                 else
+                   {
+                     user: app_spinner
+                   }
+                 end
 
       spinners.each do |name,spinner|
         @spinner = spinner

data/lib/nhkore/cli/get_cmd.rb

@@ -3,22 +3,20 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/util'
 
-
 module NHKore
 module CLI
   module GetCmd
     DEFAULT_GET_CHUNK_SIZE = 4 * 1024
     DEFAULT_GET_URL_LENGTH = 11_000_000 # Just a generous estimation used as a fallback; may be outdated.
     GET_URL_FILENAME = 'nhkore-core.zip'
-    GET_URL = "https://github.com/esotericpig/nhkore/releases/latest/download/#{GET_URL_FILENAME}"
+    GET_URL = "https://github.com/esotericpig/nhkore/releases/latest/download/#{GET_URL_FILENAME}".freeze
 
     def build_get_cmd
       app = self
@@ -27,8 +25,8 @@ module CLI
         name    'get'
         usage   'get [OPTIONS] [COMMAND]...'
         aliases :g
-        summary "Download NHKore's pre-scraped files from the latest release" \
-                " (aliases: #{app.color_alias('g')})"
+        summary "Download NHKore's pre-scraped files from the latest release " \
+                "(aliases: #{app.color_alias('g')})"
 
         description(<<-DESC)
           Download NHKore's pre-scraped files from the latest release &
@@ -41,7 +39,7 @@ module CLI
             transform: lambda { |value|
               app.check_empty_opt(:out,value)
             }
-        flag nil,:'show-url','show download URL and exit (for downloading manually)' do |value,cmd|
+        flag nil,:'show-url','show download URL and exit (for downloading manually)' do |_value,_cmd|
           puts GET_URL
           exit
         end

data/lib/nhkore/cli/news_cmd.rb

@@ -3,12 +3,12 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
+require 'fileutils'
 require 'time'
 
 require 'nhkore/datetime_parser'
@@ -18,7 +18,6 @@ require 'nhkore/news'
 require 'nhkore/search_link'
 require 'nhkore/util'
 
-
 module NHKore
 module CLI
   module NewsCmd
@@ -112,7 +111,7 @@ module CLI
           app.check_empty_opt(:url,value)
         }
 
-        run do |opts,args,cmd|
+        run do |_opts,_args,cmd|
           puts cmd.help
         end
       end
@@ -197,10 +196,8 @@ module CLI
       url = in_file.nil? ? Util.strip_web_str(@cmd_opts[:url].to_s) : in_file
       url = nil if url.empty?
 
-      if url.nil?
-        # Then we must have a links file that exists.
-        return unless check_in_file(:links,empty_ok: false)
-      end
+      # Then we must have a links file that exists.
+      return if url.nil? && !check_in_file(:links,empty_ok: false)
 
       start_spin("Scraping NHK News Web #{news_name} articles")
 
@@ -208,16 +205,14 @@ module CLI
       link_count = -1
       links = File.exist?(links_file) ? SearchLinks.load_file(links_file) : SearchLinks.new
       new_articles = [] # For --dry-run
-      news = nil
       scrape_count = 0
 
-      if File.exist?(out_file)
-        news = (type == :yasashii) ?
-          YasashiiNews.load_file(out_file,overwrite: no_sha256) :
-          FutsuuNews.load_file(out_file,overwrite: no_sha256)
-      else
-        news = (type == :yasashii) ? YasashiiNews.new : FutsuuNews.new
-      end
+      news = if File.exist?(out_file)
+               (type == :yasashii) ? YasashiiNews.load_file(out_file,overwrite: no_sha256)
+                                   : FutsuuNews.load_file(out_file,overwrite: no_sha256)
+             else
+               (type == :yasashii) ? YasashiiNews.new : FutsuuNews.new
+             end
 
       @news_article_scraper_kargs = @scraper_kargs.merge({
         datetime: datetime,
@@ -302,9 +297,9 @@ module CLI
         if show_dict
           puts @cmd_opts[:show_dict] # Updated in scrape_news_article()
         elsif dry_run
-          if new_articles.length < 1
-            raise CLIError,"scrape_count[#{scrape_count}] != new_articles[#{new_articles.length}];" \
-              ' internal code is broken'
+          if new_articles.empty?
+            raise CLIError,"scrape_count[#{scrape_count}] != new_articles[#{new_articles.length}]; " \
+                           'internal code is broken'
           elsif new_articles.length == 1
             puts new_articles.first
           else

data/lib/nhkore/cli/search_cmd.rb

@@ -3,18 +3,16 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2022 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/error'
 require 'nhkore/search_link'
 require 'nhkore/search_scraper'
 require 'nhkore/util'
 
-
 module NHKore
 module CLI
   module SearchCmd
@@ -29,7 +27,7 @@ module CLI
 
         description <<-DESC
           Search for links (using a Search Engine, etc.) to NHK News Web (Easy) &
-          save to folder: #{SearchLinks::DEFAULT_DIR}
+          save to folder: '#{SearchLinks::DEFAULT_DIR}'
         DESC
 
         option :i,:in,<<-DESC,argument: :required,transform: lambda { |value|
@@ -40,7 +38,7 @@ module CLI
         }
         option :l,:loop,'number of times to repeat the search to ensure results',argument: :required,
           transform: lambda { |value|
-            value = value.to_i
+            value = value.to_s.strip.to_i
             value = 1 if value < 1
             value
           }
@@ -68,8 +66,8 @@ module CLI
           (see '--in' option)
         DESC
 
-        run do |opts,args,cmd|
-          opts.each do |key,value|
+        run do |opts,_args,cmd|
+          opts.each do |key,_value|
             key = key.to_s
 
             if key.include?('show')
@@ -176,16 +174,15 @@ module CLI
       start_spin("Scraping #{search_type}") unless show_count
 
       is_file = !in_file.nil?
-      links = nil
       new_links = [] # For --dry-run
       url = in_file # nil will use default URL, else a file
 
-      # Load previous links for 'scraped?' vars.
-      if File.exist?(out_file)
-        links = SearchLinks.load_file(out_file)
-      else
-        links = SearchLinks.new
-      end
+      links = if File.exist?(out_file)
+                # Load previous links for 'scraped?' vars.
+                SearchLinks.load_file(out_file)
+              else
+                SearchLinks.new
+              end
 
       links_count = links.length
 

data/lib/nhkore/cli/sift_cmd.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'date'
 require 'time'
 
@@ -17,13 +16,12 @@ require 'nhkore/news'
 require 'nhkore/sifter'
 require 'nhkore/util'
 
-
 module NHKore
 module CLI
   module SiftCmd
     DEFAULT_SIFT_EXT = :csv
-    DEFAULT_SIFT_FUTSUU_FILE = "#{Sifter::DEFAULT_FUTSUU_FILE}{search.criteria}{file.ext}"
-    DEFAULT_SIFT_YASASHII_FILE = "#{Sifter::DEFAULT_YASASHII_FILE}{search.criteria}{file.ext}"
+    DEFAULT_SIFT_FUTSUU_FILE = "#{Sifter::DEFAULT_FUTSUU_FILE}{search.criteria}{file.ext}".freeze
+    DEFAULT_SIFT_YASASHII_FILE = "#{Sifter::DEFAULT_YASASHII_FILE}{search.criteria}{file.ext}".freeze
     SIFT_EXTS = %i[csv htm html json yaml yml].freeze
 
     attr_accessor :sift_datetime_text
@@ -39,8 +37,8 @@ module CLI
         name    'sift'
         usage   'sift [OPTIONS] [COMMAND]...'
         aliases :s
-        summary 'Sift NHK News Web (Easy) articles data for the frequency of words' \
-                " (aliases: #{app.color_alias('s')})"
+        summary 'Sift NHK News Web (Easy) articles data for the frequency of words ' \
+                "(aliases: #{app.color_alias('s')})"
 
         description(<<-DESC)
           Sift NHK News Web (Easy) articles data for the frequency of words &
@@ -93,11 +91,11 @@ module CLI
           to not fail on "duplicate" articles; see '#{App::NAME} news'
         DESC
         option :t,:title,'title to filter on, where search text only needs to be somewhere in the title',
-          argument: :required
+               argument: :required
         option :u,:url,'URL to filter on, where search text only needs to be somewhere in the URL',
-          argument: :required
+               argument: :required
 
-        run do |opts,args,cmd|
+        run do |_opts,_args,cmd|
           puts cmd.help
         end
       end
@@ -232,11 +230,11 @@ module CLI
       sifter.caption = "NHK News Web #{news_name}".dup
 
       if !@sift_search_criteria.nil?
-        if %i[htm html].any?(file_ext)
-          sifter.caption << " &mdash; #{Util.escape_html(@sift_search_criteria.to_s)}"
-        else
-          sifter.caption << " -- #{@sift_search_criteria}"
-        end
+        sifter.caption << if %i[htm html].any?(file_ext)
+                            " &mdash; #{Util.escape_html(@sift_search_criteria.to_s)}"
+                          else
+                            " -- #{@sift_search_criteria}"
+                          end
       end
 
       case file_ext