nhkore 0.3.4 → 0.3.9

data/lib/nhkore/scraper.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -31,16 +19,19 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class Scraper
+    extend AttrBool::Ext
+
     DEFAULT_HEADER = {
-      'user-agent' => UserAgents.sample(),
-      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp,image/apng,*/*;application/signed-exchange',
+      'user-agent' => UserAgents.sample,
+      'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp' \
+                  ',image/apng,*/*;application/signed-exchange',
       'dnt' => '1',
-    }
-    
+    }.freeze
+
     attr_accessor? :eat_cookie
     attr_accessor? :is_file
     attr_reader :kargs
@@ -49,80 +40,81 @@ module NHKore
     attr_accessor :redirect_rule
     attr_accessor :str_or_io
     attr_accessor :url
-    
+
     # +max_redirects+ defaults to 3 for safety (infinite-loop attack).
-    # 
+    #
     # All URL options: https://ruby-doc.org/stdlib-2.7.0/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
-    # 
+    #
     # Pass in +header: {}+ for the default HTTP header fields to be set.
-    # 
+    #
     # @param eat_cookie [true,false] true to set the HTTP header field 'cookie', which can be an expensive
     #                   (time-consuming) operation since it opens the URL again, but necessary for some URLs.
     # @param redirect_rule [nil,:lenient,:strict]
-    def initialize(url,eat_cookie: false,header: nil,is_file: false,max_redirects: 3,max_retries: 3,redirect_rule: :strict,str_or_io: nil,**kargs)
+    def initialize(url,eat_cookie: false,header: nil,is_file: false,max_redirects: 3,max_retries: 3,
+        redirect_rule: :strict,str_or_io: nil,**kargs)
       super()
-      
-      if !header.nil?() && !is_file
+
+      if !header.nil? && !is_file
         # Some sites (Search Engines) hate scrapers, so need HTTP header fields.
         # If this isn't enough, look at googler for more header fields to set:
         # - https://github.com/jarun/googler
         # If necessary, can use Faraday, HTTParty, or RestClient gem and
         #   pass in to str_or_io.
-        
+
         header = DEFAULT_HEADER.merge(header)
         kargs.merge!(header)
       end
-      
+
       @eat_cookie = eat_cookie
       @is_file = is_file
       @kargs = kargs
       @max_redirects = max_redirects
       @max_retries = max_retries
       @redirect_rule = redirect_rule
-      
-      open(url,str_or_io,is_file: is_file)
+
+      self.open(url,str_or_io,is_file: is_file)
     end
-    
+
     def fetch_cookie(url)
       require 'http-cookie'
-      
+
       open_url(url)
-      
+
       cookies = Array(@str_or_io.meta['set-cookie']) # nil will be []
-      
-      if !cookies.empty?()
-        jar = HTTP::CookieJar.new()
+
+      if !cookies.empty?
+        jar = HTTP::CookieJar.new
         uri = URI(url)
-        
-        cookies.each() do |cookie|
+
+        cookies.each do |cookie|
           jar.parse(cookie,uri)
         end
-        
+
         @kargs['cookie'] = HTTP::Cookie.cookie_value(jar.cookies(uri))
       end
-      
+
       return self
     end
-    
-    def html_doc()
+
+    def html_doc
       return Nokogiri::HTML(@str_or_io)
     end
-    
+
     def join_url(relative_url)
       # For a file, don't know what to do.
       # It would be unsafe to return something else;
       #   for example, it could return a lot of "../../../" to your root dir.
       return nil if @is_file
-      
-      return URI::join(@url,relative_url)
+
+      return URI.join(@url,relative_url)
     end
-    
+
     def open(url,str_or_io=nil,is_file: @is_file)
       @is_file = is_file
       @str_or_io = str_or_io
       @url = url
-      
-      if str_or_io.nil?()
+
+      if str_or_io.nil?
         if @is_file
           open_file(url)
         else
@@ -130,85 +122,86 @@ module NHKore
           open_url(url)
         end
       end
-      
+
       return self
     end
-    
+
     def open_file(file)
       @is_file = true
       @url = file
-      
+
       # NHK's website tends to always use UTF-8.
       @str_or_io = File.open(file,'rt:UTF-8',**@kargs)
-      
+
       return self
     end
-    
+
     def open_url(url)
-      max_redirects = (@max_redirects.nil?() || @max_redirects < 0) ? 10_000 : @max_redirects
-      max_retries = (@max_retries.nil?() || @max_retries < 0) ? 10_000 : @max_retries
-      
+      max_redirects = (@max_redirects.nil? || @max_redirects < 0) ? 10_000 : @max_redirects
+      max_retries = (@max_retries.nil? || @max_retries < 0) ? 10_000 : @max_retries
+
       top_uri = URI(url)
       top_domain = Util.domain(top_uri.host)
-      
+
       begin
-        # Use URI.open() instead of (Kernel.)open() for safety (code-injection attack).
-        @str_or_io = URI.open(url,redirect: false,**@kargs)
+        # Use URI().open() instead of URI.open()/(Kernel.)open() for safety (code-injection attack).
+        # Use URI() instead of URI.parse() because url can be a URI (not just a string).
+        @str_or_io = URI(url).open(redirect: false,**@kargs)
         @url = url
       rescue OpenURI::HTTPRedirect => redirect
         redirect_uri = redirect.uri
-        
+
         if (max_redirects -= 1) < 0
           raise redirect.exception("redirected to URL[#{redirect_uri}]: #{redirect}")
         end
-        
+
         case @redirect_rule
         when :lenient,:strict
           if redirect_uri.scheme != top_uri.scheme
-            raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " +
+            raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " \
               "scheme[#{top_uri.scheme}] at redirect URL[#{redirect_uri}]: #{redirect}")
           end
-          
+
           if @redirect_rule == :strict
             redirect_domain = Util.domain(redirect_uri.host)
-            
+
             if redirect_domain != top_domain
-              raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " +
+              raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " \
                 "domain[#{top_domain}] at redirect URL[#{redirect_uri}]: #{redirect}")
             end
           end
         end
-        
+
         url = redirect_uri
-        
+
         retry
       # Must come after HTTPRedirect since a subclass of HTTPError.
       rescue OpenURI::HTTPError => e
-        raise e.exception("HTTP error[#{e.to_s()}] at URL[#{url}]")
+        raise e.exception("HTTP error[#{e}] at URL[#{url}]")
       rescue SocketError => e
         if (max_retries -= 1) < 0
-          raise e.exception("Socket error[#{e.to_s()}] at URL[#{url}]")
+          raise e.exception("Socket error[#{e}] at URL[#{url}]")
         end
-        
+
         retry
       end
-      
+
       return self
     end
-    
-    def read()
-      @str_or_io = @str_or_io.read() if @str_or_io.respond_to?(:read)
-      
+
+    def read
+      @str_or_io = @str_or_io.read if @str_or_io.respond_to?(:read)
+
       return @str_or_io
     end
-    
-    def reopen()
-      return open(@url)
+
+    def reopen
+      return self.open(@url)
     end
-    
-    def rss_doc()
+
+    def rss_doc
       require 'rss'
-      
+
       return RSS::Parser.parse(@str_or_io,validate: false)
     end
   end

data/lib/nhkore/search_link.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -30,70 +18,88 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class SearchLink
-    attr_accessor :datetime
-    attr_accessor :futsuurl
+    extend AttrBool::Ext
+
+    attr_reader :datetime
+    attr_reader :futsuurl
     attr_accessor? :scraped
     attr_accessor :sha256
     attr_accessor :title
-    attr_accessor :url
-    
+    attr_reader :url
+
     def initialize(url,scraped: false)
       super()
-      
+
       @datetime = nil
       @futsuurl = nil
       @scraped = scraped
       @sha256 = sha256
       @title = nil
-      @url = url
+      self.url = url
     end
-    
+
     def encode_with(coder)
       # Order matters.
-      
-      coder[:url] = @url
+
+      coder[:url] = @url.nil? ? nil : @url.to_s
       coder[:scraped] = @scraped
-      coder[:datetime] = @datetime.nil?() ? @datetime : @datetime.iso8601()
+      coder[:datetime] = @datetime.nil? ? nil : @datetime.iso8601
       coder[:title] = @title
-      coder[:futsuurl] = @futsuurl
+      coder[:futsuurl] = @futsuurl.nil? ? nil : @futsuurl.to_s
       coder[:sha256] = @sha256
     end
-    
+
     def self.load_data(key,hash)
-      datetime = hash[:datetime]
-      
       slink = SearchLink.new(
         hash[:url],
-        scraped: hash[:scraped]
+        scraped: hash[:scraped],
       )
-      
-      slink.datetime = Util.empty_web_str?(datetime) ? nil : Time.iso8601(datetime)
+
+      slink.datetime = hash[:datetime]
       slink.futsuurl = hash[:futsuurl]
       slink.sha256 = hash[:sha256]
       slink.title = hash[:title]
-      
+
       return slink
     end
-    
+
     def update_from_article(article)
       # Don't update the url, as it may be different (e.g., http vs https).
-      
-      @datetime = article.datetime if @datetime.nil?()
-      @futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
+
+      self.datetime = article.datetime if @datetime.nil?
+      self.futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
       @scraped = true # If we have an article, it's been scraped
       @sha256 = article.sha256 if Util.empty_web_str?(@sha256)
       @title = article.title if Util.empty_web_str?(@title)
     end
-    
+
+    def datetime=(value)
+      if value.is_a?(Time)
+        @datetime = value
+      else
+        @datetime = Util.empty_web_str?(value) ? nil : Time.iso8601(value)
+      end
+    end
+
+    def futsuurl=(value)
+      # Don't store URI, store String.
+      @futsuurl = value.nil? ? nil : value.to_s
+    end
+
+    def url=(value)
+      # Don't store URI, store String.
+      @url = value.nil? ? nil : value.to_s
+    end
+
     def to_s(mini: false)
-      s = ''.dup()
-      
+      s = ''.dup
+
       s << "'#{@url}': "
-      
+
       if mini
         s << "{ scraped? #{@scraped ? 'yes' : 'NO'} }"
       else
@@ -103,84 +109,85 @@ module NHKore
         s << "\n  futsuurl: '#{@futsuurl}'"
         s << "\n  sha256:   '#{@sha256}'"
       end
-      
+
       return s
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class SearchLinks
     include Fileable
-    
+
     DEFAULT_DIR = Util::CORE_DIR
-    
+
     DEFAULT_FUTSUU_FILENAME = 'links_nhk_news_web_regular.yml'
     DEFAULT_YASASHII_FILENAME = 'links_nhk_news_web_easy.yml'
-    
+
     def self.build_file(filename)
       return File.join(DEFAULT_DIR,filename)
     end
-    
+
     DEFAULT_FUTSUU_FILE = build_file(DEFAULT_FUTSUU_FILENAME)
     DEFAULT_YASASHII_FILE = build_file(DEFAULT_YASASHII_FILENAME)
-    
+
     attr_reader :links
-    
-    def initialize()
+
+    def initialize
       super()
-      
+
       @links = {}
     end
-    
+
     def add_link(link)
-      return self if @links.key?(link.url)
-      
-      @links[link.url] = link
-      
+      url = link.url.nil? ? nil : link.url.to_s
+
+      return self if @links.key?(url)
+
+      @links[url] = link
+
       return self
     end
-    
+
     def each(&block)
       return @links.each(&block)
     end
-    
+
     def encode_with(coder)
       # Order matters.
-      
+
       coder[:links] = @links
     end
-    
+
     def self.load_data(data,file: nil,**kargs)
       data = Util.load_yaml(data,file: file)
-      
+
       links = data[:links]
-      
-      slinks = SearchLinks.new()
-      
-      if !links.nil?()
-        links.each() do |key,hash|
-          key = key.to_s() # Change from a symbol
-          slinks.links[key] = SearchLink.load_data(key,hash)
-        end
+
+      slinks = SearchLinks.new
+
+      links&.each() do |key,hash|
+        key = key.to_s unless key.nil?
+        slinks.links[key] = SearchLink.load_data(key,hash)
       end
-      
+
       return slinks
     end
-    
+
     def [](url)
       url = url.url if url.respond_to?(:url)
-      
+      url = url.to_s unless url.nil?
+
       return @links[url]
     end
-    
-    def length()
+
+    def length
       return @links.length
     end
-    
-    def to_s()
+
+    def to_s
       return Util.dump_yaml(self)
     end
   end