nhkore 0.3.7 → 0.3.11

data/lib/nhkore/news.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -29,155 +17,153 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class News
     include Fileable
-    
+
     DEFAULT_DIR = Util::CORE_DIR
-    FAVORED_URL = /https\:/i
-    
+    FAVORED_URL = /https:/i.freeze
+
     attr_reader :articles
     attr_reader :sha256s
-    
-    def initialize()
+
+    def initialize
       super()
-      
+
       @articles = {}
       @sha256s = {}
     end
-    
+
     def add_article(article,key: nil,overwrite: false)
       url = article.url
-      url = url.to_s() unless url.nil?()
-      
-      key = key.nil?() ? url : key.to_s()
-      
+      url = url.to_s unless url.nil?
+
+      key = key.nil? ? url : key.to_s
+
       if !overwrite
         raise ArgumentError,"duplicate article[#{key}] in articles" if @articles.key?(key)
         raise ArgumentError,"duplicate sha256[#{article.sha256}] in articles" if @sha256s.key?(article.sha256)
       end
-      
+
       @articles[key] = article
       @sha256s[article.sha256] = url
-      
+
       return self
     end
-    
+
     def self.build_file(filename)
       return File.join(DEFAULT_DIR,filename)
     end
-    
+
     def encode_with(coder)
       # Order matters.
       # Don't output @sha256s.
-      
+
       coder[:articles] = @articles
     end
-    
+
     def self.load_data(data,article_class: Article,file: nil,news_class: News,overwrite: false,**kargs)
       data = Util.load_yaml(data,file: file)
-      
+
       articles = data[:articles]
-      
-      news = news_class.new()
-      
-      if !articles.nil?()
-        articles.each() do |key,hash|
-          key = key.to_s() # Change from a symbol
-          news.add_article(article_class.load_data(key,hash),key: key,overwrite: overwrite)
-        end
+
+      news = news_class.new
+
+      articles&.each() do |key,hash|
+        key = key.to_s # Change from a symbol
+        news.add_article(article_class.load_data(key,hash),key: key,overwrite: overwrite)
       end
-      
+
       return news
     end
-    
+
     def update_article(article,url)
-      url = url.to_s() unless url.nil?()
-      
+      url = url.to_s unless url.nil?
+
       # Favor https.
-      return if article.url.to_s() =~ FAVORED_URL
+      return if article.url.to_s =~ FAVORED_URL
       return if url !~ FAVORED_URL
-      
+
       @articles.delete(article.url) # Probably no to_s() here
       @articles[url] = article
       article.url = url
     end
-    
+
     def article(key)
-      key = key.to_s() unless key.nil?()
-      
+      key = key.to_s unless key.nil?
+
       return @articles[key]
     end
-    
+
     def article_with_sha256(sha256)
       article = nil
-      
-      @articles.values().each() do |a|
+
+      @articles.each_value do |a|
         if a.sha256 == sha256
           article = a
-          
+
           break
         end
       end
-      
+
       return article
     end
-    
+
     def article?(key)
-      key = key.to_s() unless key.nil?()
-      
+      key = key.to_s unless key.nil?
+
       return @articles.key?(key)
     end
-    
+
     def sha256?(sha256)
       return @sha256s.key?(sha256)
     end
-    
-    def to_s()
+
+    def to_s
       # Put each Word on one line (flow/inline style).
       return Util.dump_yaml(self,flow_level: 8)
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class FutsuuNews < News
     DEFAULT_FILENAME = 'nhk_news_web_regular.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
-    
+
     def self.load_data(data,**kargs)
       return News.load_data(data,article_class: Article,news_class: FutsuuNews,**kargs)
     end
-    
+
     def self.load_file(file=DEFAULT_FILE,**kargs)
       return News.load_file(file,article_class: Article,news_class: FutsuuNews,**kargs)
     end
-    
+
     def save_file(file=DEFAULT_FILE,**kargs)
       super(file,**kargs)
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class YasashiiNews < News
     DEFAULT_FILENAME = 'nhk_news_web_easy.yml'
     DEFAULT_FILE = build_file(DEFAULT_FILENAME)
-    
+
     def self.load_data(data,**kargs)
       return News.load_data(data,article_class: Article,news_class: YasashiiNews,**kargs)
     end
-    
+
     def self.load_file(file=DEFAULT_FILE,**kargs)
       return News.load_file(file,article_class: Article,news_class: YasashiiNews,**kargs)
     end
-    
+
     def save_file(file=DEFAULT_FILE,**kargs)
       super(file,**kargs)
     end

data/lib/nhkore/polisher.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -26,28 +14,28 @@ require 'nhkore/word'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class Polisher
     def begin_polish(str)
       return str
     end
-    
+
     def polish(str)
       str = begin_polish(str)
       str = end_polish(str)
-      
+
       return str
     end
-    
+
     def self.polish_any(obj,polishers)
-      return nil if obj.nil?()
-      
+      return nil if obj.nil?
+
       polishers = Array(polishers)
-      
-      return obj if polishers.empty?()
-      
+
+      return obj if polishers.empty?
+
       if obj.is_a?(Word)
         obj = Word.new(
           kana: polish_any(obj.kana,polishers),
@@ -55,17 +43,17 @@ module NHKore
           word: obj
         )
       else # String
-        polishers.each() do |polisher|
+        polishers.each do |polisher|
           obj = polisher.polish(obj)
         end
       end
-      
+
       return obj
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BasicPolisher < Polisher
@@ -74,18 +62,18 @@ module NHKore
       # - Yunibaasaru・Sutajio・Japan
       # Keep numbers next to kanji/kana, else the below kana won't make sense:
       # - Word { kanji: ２０日, kana: はつか }
-      
+
       str = str.gsub(/[^[[:alnum:]]・]/,'')
-      
+
       # Numbers/dots by themselves (without kanji/kana) should be ignored (empty).
-      str = '' if str.gsub(/[[[:digit:]]・]+/,'').empty?()
-      
+      str = '' if str.gsub(/[[[:digit:]]・]+/,'').empty?
+
       return str
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BestPolisher < BasicPolisher

data/lib/nhkore/scraper.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -31,18 +19,19 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class Scraper
     extend AttrBool::Ext
-    
+
     DEFAULT_HEADER = {
-      'user-agent' => UserAgents.sample(),
-      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp,image/apng,*/*;application/signed-exchange',
+      'user-agent' => UserAgents.sample,
+      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp' \
+                  ',image/apng,*/*;application/signed-exchange',
       'dnt' => '1',
-    }
-    
+    }.freeze
+
     attr_accessor? :eat_cookie
     attr_accessor? :is_file
     attr_reader :kargs
@@ -51,80 +40,81 @@ module NHKore
     attr_accessor :redirect_rule
     attr_accessor :str_or_io
     attr_accessor :url
-    
+
     # +max_redirects+ defaults to 3 for safety (infinite-loop attack).
-    # 
+    #
     # All URL options: https://ruby-doc.org/stdlib-2.7.0/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
-    # 
+    #
     # Pass in +header: {}+ for the default HTTP header fields to be set.
-    # 
+    #
     # @param eat_cookie [true,false] true to set the HTTP header field 'cookie', which can be an expensive
     #                   (time-consuming) operation since it opens the URL again, but necessary for some URLs.
     # @param redirect_rule [nil,:lenient,:strict]
-    def initialize(url,eat_cookie: false,header: nil,is_file: false,max_redirects: 3,max_retries: 3,redirect_rule: :strict,str_or_io: nil,**kargs)
+    def initialize(url,eat_cookie: false,header: nil,is_file: false,max_redirects: 3,max_retries: 3,
+        redirect_rule: :strict,str_or_io: nil,**kargs)
       super()
-      
-      if !header.nil?() && !is_file
+
+      if !header.nil? && !is_file
         # Some sites (Search Engines) hate scrapers, so need HTTP header fields.
         # If this isn't enough, look at googler for more header fields to set:
         # - https://github.com/jarun/googler
         # If necessary, can use Faraday, HTTParty, or RestClient gem and
         #   pass in to str_or_io.
-        
+
         header = DEFAULT_HEADER.merge(header)
         kargs.merge!(header)
       end
-      
+
       @eat_cookie = eat_cookie
       @is_file = is_file
       @kargs = kargs
       @max_redirects = max_redirects
       @max_retries = max_retries
       @redirect_rule = redirect_rule
-      
-      open(url,str_or_io,is_file: is_file)
+
+      self.open(url,str_or_io,is_file: is_file)
     end
-    
+
     def fetch_cookie(url)
       require 'http-cookie'
-      
+
       open_url(url)
-      
+
       cookies = Array(@str_or_io.meta['set-cookie']) # nil will be []
-      
-      if !cookies.empty?()
-        jar = HTTP::CookieJar.new()
+
+      if !cookies.empty?
+        jar = HTTP::CookieJar.new
         uri = URI(url)
-        
-        cookies.each() do |cookie|
+
+        cookies.each do |cookie|
           jar.parse(cookie,uri)
         end
-        
+
         @kargs['cookie'] = HTTP::Cookie.cookie_value(jar.cookies(uri))
       end
-      
+
       return self
     end
-    
-    def html_doc()
+
+    def html_doc
       return Nokogiri::HTML(@str_or_io)
     end
-    
+
     def join_url(relative_url)
       # For a file, don't know what to do.
       # It would be unsafe to return something else;
       #   for example, it could return a lot of "../../../" to your root dir.
       return nil if @is_file
-      
-      return URI::join(@url,relative_url)
+
+      return URI.join(@url,relative_url)
     end
-    
+
     def open(url,str_or_io=nil,is_file: @is_file)
       @is_file = is_file
       @str_or_io = str_or_io
       @url = url
-      
-      if str_or_io.nil?()
+
+      if str_or_io.nil?
         if @is_file
           open_file(url)
         else
@@ -132,85 +122,86 @@ module NHKore
           open_url(url)
         end
       end
-      
+
       return self
     end
-    
+
     def open_file(file)
       @is_file = true
       @url = file
-      
+
       # NHK's website tends to always use UTF-8.
       @str_or_io = File.open(file,'rt:UTF-8',**@kargs)
-      
+
       return self
     end
-    
+
     def open_url(url)
-      max_redirects = (@max_redirects.nil?() || @max_redirects < 0) ? 10_000 : @max_redirects
-      max_retries = (@max_retries.nil?() || @max_retries < 0) ? 10_000 : @max_retries
-      
+      max_redirects = (@max_redirects.nil? || @max_redirects < 0) ? 10_000 : @max_redirects
+      max_retries = (@max_retries.nil? || @max_retries < 0) ? 10_000 : @max_retries
+
       top_uri = URI(url)
       top_domain = Util.domain(top_uri.host)
-      
+
       begin
-        # Use URI.open() instead of (Kernel.)open() for safety (code-injection attack).
-        @str_or_io = URI.open(url,redirect: false,**@kargs)
+        # Use URI().open() instead of URI.open()/(Kernel.)open() for safety (code-injection attack).
+        # Use URI() instead of URI.parse() because url can be a URI (not just a string).
+        @str_or_io = URI(url).open(redirect: false,**@kargs)
         @url = url
       rescue OpenURI::HTTPRedirect => redirect
         redirect_uri = redirect.uri
-        
+
         if (max_redirects -= 1) < 0
           raise redirect.exception("redirected to URL[#{redirect_uri}]: #{redirect}")
         end
-        
+
         case @redirect_rule
         when :lenient,:strict
           if redirect_uri.scheme != top_uri.scheme
-            raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " +
+            raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " \
               "scheme[#{top_uri.scheme}] at redirect URL[#{redirect_uri}]: #{redirect}")
           end
-          
+
           if @redirect_rule == :strict
             redirect_domain = Util.domain(redirect_uri.host)
-            
+
             if redirect_domain != top_domain
-              raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " +
+              raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " \
                 "domain[#{top_domain}] at redirect URL[#{redirect_uri}]: #{redirect}")
             end
           end
         end
-        
+
         url = redirect_uri
-        
+
         retry
       # Must come after HTTPRedirect since a subclass of HTTPError.
       rescue OpenURI::HTTPError => e
-        raise e.exception("HTTP error[#{e.to_s()}] at URL[#{url}]")
+        raise e.exception("HTTP error[#{e}] at URL[#{url}]")
       rescue SocketError => e
         if (max_retries -= 1) < 0
-          raise e.exception("Socket error[#{e.to_s()}] at URL[#{url}]")
+          raise e.exception("Socket error[#{e}] at URL[#{url}]")
         end
-        
+
         retry
       end
-      
+
       return self
     end
-    
-    def read()
-      @str_or_io = @str_or_io.read() if @str_or_io.respond_to?(:read)
-      
+
+    def read
+      @str_or_io = @str_or_io.read if @str_or_io.respond_to?(:read)
+
       return @str_or_io
     end
-    
-    def reopen()
-      return open(@url)
+
+    def reopen
+      return self.open(@url)
     end
-    
-    def rss_doc()
+
+    def rss_doc
       require 'rss'
-      
+
       return RSS::Parser.parse(@str_or_io,validate: false)
     end
   end