nhkore 0.3.17 → 0.3.19

data/lib/nhkore/datetime_parser.rb

@@ -3,19 +3,17 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'attr_bool'
 require 'date'
 require 'time'
 
 require 'nhkore/util'
 
-
 module NHKore
   class DatetimeParser
     extend AttrBool::Ext
@@ -55,10 +53,10 @@ module NHKore
             # Assume this millennium.
             # So if the current year is 2200, and year is 150,
             # then it will be 2000 + 150 = 2150.
-          else
+          elsif millennium >= 1000
             # Assume previous millennium (2000 -> 1000),
             # so year 999 will become 1999.
-            millennium -= 1000 if millennium >= 1000
+            millennium -= 1000
           end
 
           year = millennium + year
@@ -155,7 +153,7 @@ module NHKore
 
     attr_reader? :min_or_max
 
-    def initialize(year=nil,month=nil,day=nil,hour=nil,min=nil,sec=nil)
+    def initialize(year = nil,month = nil,day = nil,hour = nil,min = nil,sec = nil)
       super()
 
       set!(year,month,day,hour,min,sec)
@@ -185,66 +183,66 @@ module NHKore
         @sec = other.sec unless @has_sec
         has_small = true
       else
-        if has_small
-          @sec = jst_now.sec
-        else
-          @sec = is_from ? 0 : 59
-        end
+        @sec = if has_small
+                 jst_now.sec
+               else
+                 is_from ? 0 : 59
+               end
       end
 
       if @has_min || other.has_min?
         @min = other.min unless @has_min
         has_small = true
       else
-        if has_small
-          @min = jst_now.min
-        else
-          @min = is_from ? 0 : 59
-        end
+        @min = if has_small
+                 jst_now.min
+               else
+                 is_from ? 0 : 59
+               end
       end
 
       if @has_hour || other.has_hour?
         @hour = other.hour unless @has_hour
         has_small = true
       else
-        if has_small
-          @hour = jst_now.hour
-        else
-          @hour = is_from ? 0 : 23
-        end
+        @hour = if has_small
+                  jst_now.hour
+                else
+                  is_from ? 0 : 23
+                end
       end
 
       if @has_day || other.has_day?
         @day = other.day unless @has_day
         has_small = true
       else
-        if has_small
-          @day = jst_now.day
-        else
-          @day = is_from ? 1 : :last_day
-        end
+        @day = if has_small
+                 jst_now.day
+               else
+                 is_from ? 1 : :last_day
+               end
       end
 
       if @has_month || other.has_month?
         @month = other.month unless @has_month
         has_small = true
       else
-        if has_small
-          @month = jst_now.month
-        else
-          @month = is_from ? 1 : 12
-        end
+        @month = if has_small
+                   jst_now.month
+                 else
+                   is_from ? 1 : 12
+                 end
       end
 
       if @has_year || other.has_year?
         @year = other.year unless @has_year
         has_small = true # rubocop:disable Lint/UselessAssignment
       else
-        if has_small
-          @year = jst_now.year
-        else
-          @year = is_from ? Util::MIN_SANE_YEAR : jst_now.year
-        end
+        @year = if has_small
+                  jst_now.year
+                else
+                  is_from ? Util::MIN_SANE_YEAR : jst_now.year
+                end
       end
 
       # Must be after setting @year & @month.
@@ -289,7 +287,7 @@ module NHKore
       return self
     end
 
-    def set!(year=nil,month=nil,day=nil,hour=nil,min=nil,sec=nil)
+    def set!(year = nil,month = nil,day = nil,hour = nil,min = nil,sec = nil)
       @year = year
       @month = month
       @day = day

data/lib/nhkore/defn.rb

@@ -3,18 +3,16 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nokogiri'
 
 require 'nhkore/util'
 require 'nhkore/word'
 
-
 module NHKore
   class Defn
     attr_reader :hyoukis
@@ -35,7 +33,7 @@ module NHKore
 
       hyoukis = hash['hyouki']
 
-      hyoukis&.each() do |hyouki|
+      hyoukis&.each do |hyouki|
         next if hyouki.nil?
         next if (hyouki = Util.strip_web_str(hyouki)).empty?
 

data/lib/nhkore/dict.rb

@@ -3,16 +3,14 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/entry'
 require 'nhkore/error'
 
-
 module NHKore
   class Dict
     attr_reader :entries

data/lib/nhkore/dict_scraper.rb

@@ -3,18 +3,16 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2022 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/dict'
 require 'nhkore/error'
 require 'nhkore/scraper'
 require 'nhkore/util'
 
-
 module NHKore
   class DictScraper < Scraper
     attr_accessor :missingno

data/lib/nhkore/entry.rb

@@ -3,16 +3,14 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/defn'
 require 'nhkore/util'
 
-
 module NHKore
   class Entry
     HYOUKI_SEP = '・'

data/lib/nhkore/error.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 module NHKore
   class Error < ::StandardError; end
 

data/lib/nhkore/fileable.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 module NHKore
   module Fileable
     def self.included(mod)

data/lib/nhkore/lib.rb

@@ -3,11 +3,15 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
+###
+# Include this file to only require the files needed to use this
+# Gem as a library (i.e., don't include CLI-related files).
+###
 
 require 'nhkore/article'
 require 'nhkore/article_scraper'
@@ -27,18 +31,7 @@ require 'nhkore/search_link'
 require 'nhkore/search_scraper'
 require 'nhkore/sifter'
 require 'nhkore/splitter'
-require 'nhkore/user_agents'
 require 'nhkore/util'
 require 'nhkore/variator'
 require 'nhkore/version'
 require 'nhkore/word'
-
-
-module NHKore
-  ###
-  # Include this file to only require the files needed to use this
-  # Gem as a library (i.e., don't include CLI-related files).
-  ###
-  module Lib
-  end
-end

data/lib/nhkore/missingno.rb

@@ -3,15 +3,13 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/util'
 
-
 module NHKore
   class Missingno
     attr_reader :kanas

data/lib/nhkore/news.rb

@@ -3,18 +3,16 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/article'
 require 'nhkore/error'
 require 'nhkore/fileable'
 require 'nhkore/util'
 
-
 module NHKore
   class News
     include Fileable
@@ -60,14 +58,14 @@ module NHKore
       coder[:articles] = @articles
     end
 
-    def self.load_data(data,article_class: Article,file: nil,news_class: News,overwrite: false,**kargs)
+    def self.load_data(data,article_class: Article,file: nil,news_class: News,overwrite: false,**_kargs)
       data = Util.load_yaml(data,file: file)
 
       articles = data[:articles]
 
       news = news_class.new
 
-      articles&.each() do |key,hash|
+      articles&.each do |key,hash|
         key = key.to_s # Change from a symbol
         news.add_article(article_class.load_data(key,hash),key: key,overwrite: overwrite)
       end
@@ -99,7 +97,6 @@ module NHKore
       @articles.each_value do |a|
         if a.sha256 == sha256
           article = a
-
           break
         end
       end
@@ -131,11 +128,11 @@ module NHKore
       return News.load_data(data,article_class: Article,news_class: FutsuuNews,**kargs)
     end
 
-    def self.load_file(file=DEFAULT_FILE,**kargs)
+    def self.load_file(file = DEFAULT_FILE,**kargs)
       return News.load_file(file,article_class: Article,news_class: FutsuuNews,**kargs)
     end
 
-    def save_file(file=DEFAULT_FILE,**kargs)
+    def save_file(file = DEFAULT_FILE,**kargs)
       super
     end
   end
@@ -148,11 +145,11 @@ module NHKore
       return News.load_data(data,article_class: Article,news_class: YasashiiNews,**kargs)
     end
 
-    def self.load_file(file=DEFAULT_FILE,**kargs)
+    def self.load_file(file = DEFAULT_FILE,**kargs)
       return News.load_file(file,article_class: Article,news_class: YasashiiNews,**kargs)
     end
 
-    def save_file(file=DEFAULT_FILE,**kargs)
+    def save_file(file = DEFAULT_FILE,**kargs)
       super
     end
   end

data/lib/nhkore/polisher.rb

@@ -3,15 +3,13 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'nhkore/word'
 
-
 module NHKore
   class Polisher
     def begin_polish(str)

data/lib/nhkore/scraper.rb

@@ -3,30 +3,36 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
+require 'ronin/web/user_agents'
 
 require 'nhkore/error'
-require 'nhkore/user_agents'
 require 'nhkore/util'
 
-
 module NHKore
   class Scraper
     extend AttrBool::Ext
 
     DEFAULT_HEADER = {
-      'user-agent' => UserAgents.sample,
-      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp' \
-                  ',image/apng,*/*;application/signed-exchange',
+      # See for better ones:
+      # - https://www.useragentstring.com/pages/Chrome/
+      'user-agent' => Ronin::Web::UserAgents.random,
+
+      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;' \
+                  'q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+      'accept-language' => 'en;q=0.9,ja-JP;q=0.8,ja',
+      'cache-control' => 'max-age=0',
       'dnt' => '1',
+      'ect' => '4g',
+      'priority' => 'u=0, i',
+      'upgrade-insecure-requests' => '1',
     }.freeze
 
     attr_accessor? :eat_cookie
@@ -48,7 +54,7 @@ module NHKore
     #                   (time-consuming) operation since it opens the URL again, but necessary for some URLs.
     # @param redirect_rule [nil,:lenient,:strict]
     def initialize(url,eat_cookie: false,header: nil,is_file: false,max_redirects: 3,max_retries: 3,
-        redirect_rule: :strict,str_or_io: nil,**kargs)
+                   redirect_rule: :strict,str_or_io: nil,**kargs)
       super()
 
       if !header.nil? && !is_file
@@ -106,7 +112,7 @@ module NHKore
       return URI.join(@url,relative_url)
     end
 
-    def open(url,str_or_io=nil,is_file: @is_file)
+    def open(url,str_or_io = nil,is_file: @is_file)
       @is_file = is_file
       @str_or_io = str_or_io
       @url = url
@@ -155,16 +161,20 @@ module NHKore
         case @redirect_rule
         when :lenient,:strict
           if redirect_uri.scheme != top_uri.scheme
-            raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " \
-              "scheme[#{top_uri.scheme}] at redirect URL[#{redirect_uri}]: #{redirect}")
+            raise redirect.exception(
+              "redirect scheme[#{redirect_uri.scheme}] does not match original " \
+              "scheme[#{top_uri.scheme}] at redirect URL[#{redirect_uri}]: #{redirect}"
+            )
           end
 
           if @redirect_rule == :strict
             redirect_domain = Util.domain(redirect_uri.host)
 
             if redirect_domain != top_domain
-              raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " \
-                "domain[#{top_domain}] at redirect URL[#{redirect_uri}]: #{redirect}")
+              raise redirect.exception(
+                "redirect domain[#{redirect_domain}] does not match original " \
+                "domain[#{top_domain}] at redirect URL[#{redirect_uri}]: #{redirect}"
+              )
             end
           end
         end

data/lib/nhkore/search_link.rb

@@ -3,19 +3,17 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'attr_bool'
 require 'time'
 
 require 'nhkore/fileable'
 require 'nhkore/util'
 
-
 module NHKore
   class SearchLink
     extend AttrBool::Ext
@@ -49,7 +47,7 @@ module NHKore
       coder[:sha256] = @sha256
     end
 
-    def self.load_data(key,hash)
+    def self.load_data(_key,hash)
       slink = SearchLink.new(
         hash[:url],
         scraped: hash[:scraped],
@@ -74,11 +72,11 @@ module NHKore
     end
 
     def datetime=(value)
-      if value.is_a?(Time)
-        @datetime = value
-      else
-        @datetime = Util.empty_web_str?(value) ? nil : Time.iso8601(value)
-      end
+      @datetime = if value.is_a?(Time)
+                    value
+                  else
+                    Util.empty_web_str?(value) ? nil : Time.iso8601(value)
+                  end
     end
 
     def futsuurl=(value)
@@ -143,8 +141,8 @@ module NHKore
       return self
     end
 
-    def each(&block)
-      return @links.each(&block)
+    def each(&)
+      return @links.each(&)
     end
 
     def encode_with(coder)
@@ -153,14 +151,14 @@ module NHKore
       coder[:links] = @links
     end
 
-    def self.load_data(data,file: nil,**kargs)
+    def self.load_data(data,file: nil,**_kargs)
       data = Util.load_yaml(data,file: file)
 
       links = data[:links]
 
       slinks = SearchLinks.new
 
-      links&.each() do |key,hash|
+      links&.each do |key,hash|
         key = key.to_s unless key.nil?
         slinks.links[key] = SearchLink.load_data(key,hash)
       end

data/lib/nhkore/search_scraper.rb

@@ -3,12 +3,11 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020 Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
-
 require 'net/http'
 require 'uri'
 
@@ -17,7 +16,6 @@ require 'nhkore/scraper'
 require 'nhkore/search_link'
 require 'nhkore/util'
 
-
 module NHKore
   class SearchScraper < Scraper
     DEFAULT_RESULT_COUNT = 100
@@ -60,8 +58,10 @@ module NHKore
 
     # Example: https://www3.nhk.or.jp/news/easy/k10014150691000/k10014150691000.html
     def fetch_valid_link?(link)
-      uri = begin
-        URI(link)
+      uri = nil
+
+      begin
+        uri = URI(link)
       rescue StandardError
         return false # Bad URL.
       end
@@ -111,29 +111,40 @@ module NHKore
       super(url,**kargs)
     end
 
-    def self.build_url(site,count: DEFAULT_RESULT_COUNT,**kargs)
+    # FIXME: Bing no longer allows `count`.
+    # rubocop:disable Lint/UnusedMethodArgument
+    def self.build_url(site,count: DEFAULT_RESULT_COUNT,**_kargs)
       url = ''.dup
 
       url << 'https://www.bing.com/search?'
       url << URI.encode_www_form(
         q: "site:#{site}",
-        count: count
+        qs: 'n',
+        sp: '-1',
+        lq: '0',
+        pq: "site:#{site}",
+        sc: '1-25',
+        sk: '',
+        first: '1',
+        FORM: 'PERE',
       )
 
       return url
     end
+    # rubocop:enable Lint/UnusedMethodArgument
 
-    def scrape(slinks,page=NextPage.new())
-      next_page,link_count = scrape_html(slinks,page)
+    def scrape(slinks,page = NextPage.new())
+      next_page,_link_count = scrape_html(slinks,page)
 
-      if link_count <= 0
-        scrape_rss(slinks,page,next_page)
-      end
+      # FIXME: Bing no longer allows RSS pages after the first page.
+      # if link_count <= 0
+      #   scrape_rss(slinks,page,next_page)
+      # end
 
       return next_page
     end
 
-    def scrape_html(slinks,page,next_page=NextPage.new())
+    def scrape_html(slinks,page,next_page = NextPage.new())
       doc = html_doc
       link_count = 0
 
@@ -145,7 +156,7 @@ module NHKore
 
         next if ignore_link?(href)
 
-        if (md = href.match(/first=(\d+)/))
+        if (md = href.match(/first=(\d+)/i)) && href =~ /FORM=PERE/i
           count = md[1].to_i
 
           if count > page.count && (next_page.count < 0 || count < next_page.count)
@@ -161,7 +172,7 @@ module NHKore
       return [next_page,link_count]
     end
 
-    def scrape_rss(slinks,page,next_page=NextPage.new())
+    def scrape_rss(slinks,page,next_page = NextPage.new())
       link_count = 0
 
       if !@is_file