nhkore 0.3.2 → 0.3.7

data/lib/nhkore/dict_scraper.rb

@@ -44,7 +44,7 @@ module NHKore
     end
     
     def self.parse_url(url,basename: nil)
-      url = Util.strip_web_str(url)
+      url = Util.strip_web_str(url.to_s())
       
       raise ParseError,"cannot parse dictionary URL from URL[#{url}]" if url.empty?()
       

data/lib/nhkore/lib.rb

@@ -24,6 +24,7 @@
 require 'nhkore/article'
 require 'nhkore/article_scraper'
 require 'nhkore/cleaner'
+require 'nhkore/datetime_parser'
 require 'nhkore/defn'
 require 'nhkore/dict'
 require 'nhkore/dict_scraper'

data/lib/nhkore/news.rb

@@ -49,7 +49,10 @@ module NHKore
     end
     
     def add_article(article,key: nil,overwrite: false)
-      key = article.url if key.nil?()
+      url = article.url
+      url = url.to_s() unless url.nil?()
+      
+      key = key.nil?() ? url : key.to_s()
       
       if !overwrite
         raise ArgumentError,"duplicate article[#{key}] in articles" if @articles.key?(key)
@@ -57,7 +60,7 @@ module NHKore
       end
       
       @articles[key] = article
-      @sha256s[article.sha256] = article.url
+      @sha256s[article.sha256] = url
       
       return self
     end
@@ -91,16 +94,20 @@ module NHKore
     end
     
     def update_article(article,url)
+      url = url.to_s() unless url.nil?()
+      
       # Favor https.
-      return if article.url =~ FAVORED_URL
+      return if article.url.to_s() =~ FAVORED_URL
       return if url !~ FAVORED_URL
       
-      @articles.delete(article.url)
+      @articles.delete(article.url) # Probably no to_s() here
       @articles[url] = article
       article.url = url
     end
     
     def article(key)
+      key = key.to_s() unless key.nil?()
+      
       return @articles[key]
     end
     
@@ -119,6 +126,8 @@ module NHKore
     end
     
     def article?(key)
+      key = key.to_s() unless key.nil?()
+      
       return @articles.key?(key)
     end
     

data/lib/nhkore/scraper.rb

@@ -21,6 +21,7 @@
 #++
 
 
+require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
 
@@ -34,14 +35,16 @@ module NHKore
   # @since  0.2.0
   ###
   class Scraper
+    extend AttrBool::Ext
+    
     DEFAULT_HEADER = {
       'user-agent' => UserAgents.sample(),
       'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp,image/apng,*/*;application/signed-exchange',
       'dnt' => '1',
     }
     
-    attr_accessor :eat_cookie
-    attr_accessor :is_file
+    attr_accessor? :eat_cookie
+    attr_accessor? :is_file
     attr_reader :kargs
     attr_accessor :max_redirects
     attr_accessor :max_retries
@@ -49,9 +52,6 @@ module NHKore
     attr_accessor :str_or_io
     attr_accessor :url
     
-    alias_method :eat_cookie?,:eat_cookie
-    alias_method :is_file?,:is_file
-    
     # +max_redirects+ defaults to 3 for safety (infinite-loop attack).
     # 
     # All URL options: https://ruby-doc.org/stdlib-2.7.0/libdoc/open-uri/rdoc/OpenURI/OpenRead.html

data/lib/nhkore/search_link.rb

@@ -21,6 +21,7 @@
 #++
 
 
+require 'attr_bool'
 require 'time'
 
 require 'nhkore/fileable'
@@ -33,14 +34,14 @@ module NHKore
   # @since  0.2.0
   ###
   class SearchLink
-    attr_accessor :datetime
-    attr_accessor :futsuurl
-    attr_accessor :scraped
+    extend AttrBool::Ext
+    
+    attr_reader :datetime
+    attr_reader :futsuurl
+    attr_accessor? :scraped
     attr_accessor :sha256
     attr_accessor :title
-    attr_accessor :url
-    
-    alias_method :scraped?,:scraped
+    attr_reader :url
     
     def initialize(url,scraped: false)
       super()
@@ -50,29 +51,27 @@ module NHKore
       @scraped = scraped
       @sha256 = sha256
       @title = nil
-      @url = url
+      self.url = url
     end
     
     def encode_with(coder)
       # Order matters.
       
-      coder[:url] = @url
+      coder[:url] = @url.nil?() ? nil : @url.to_s()
       coder[:scraped] = @scraped
-      coder[:datetime] = @datetime.nil?() ? @datetime : @datetime.iso8601()
+      coder[:datetime] = @datetime.nil?() ? nil : @datetime.iso8601()
       coder[:title] = @title
-      coder[:futsuurl] = @futsuurl
+      coder[:futsuurl] = @futsuurl.nil?() ? nil : @futsuurl.to_s()
       coder[:sha256] = @sha256
     end
     
     def self.load_data(key,hash)
-      datetime = hash[:datetime]
-      
       slink = SearchLink.new(
         hash[:url],
-        scraped: hash[:scraped]
+        scraped: hash[:scraped],
       )
       
-      slink.datetime = Util.empty_web_str?(datetime) ? nil : Time.iso8601(datetime)
+      slink.datetime = hash[:datetime]
       slink.futsuurl = hash[:futsuurl]
       slink.sha256 = hash[:sha256]
       slink.title = hash[:title]
@@ -83,13 +82,31 @@ module NHKore
     def update_from_article(article)
       # Don't update the url, as it may be different (e.g., http vs https).
       
-      @datetime = article.datetime if @datetime.nil?()
-      @futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
+      self.datetime = article.datetime if @datetime.nil?()
+      self.futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
       @scraped = true # If we have an article, it's been scraped
       @sha256 = article.sha256 if Util.empty_web_str?(@sha256)
       @title = article.title if Util.empty_web_str?(@title)
     end
     
+    def datetime=(value)
+      if value.is_a?(Time)
+        @datetime = value
+      else
+        @datetime = Util.empty_web_str?(value) ? nil : Time.iso8601(value)
+      end
+    end
+    
+    def futsuurl=(value)
+      # Don't store URI, store String.
+      @futsuurl = value.nil?() ? nil : value.to_s()
+    end
+    
+    def url=(value)
+      # Don't store URI, store String.
+      @url = value.nil?() ? nil : value.to_s()
+    end
+    
     def to_s(mini: false)
       s = ''.dup()
       
@@ -137,9 +154,11 @@ module NHKore
     end
     
     def add_link(link)
-      return self if @links.key?(link.url)
+      url = link.url.nil?() ? nil : link.url.to_s()
+      
+      return self if @links.key?(url)
       
-      @links[link.url] = link
+      @links[url] = link
       
       return self
     end
@@ -163,7 +182,7 @@ module NHKore
       
       if !links.nil?()
         links.each() do |key,hash|
-          key = key.to_s() # Change from a symbol
+          key = key.to_s() unless key.nil?()
           slinks.links[key] = SearchLink.load_data(key,hash)
         end
       end
@@ -173,6 +192,7 @@ module NHKore
     
     def [](url)
       url = url.url if url.respond_to?(:url)
+      url = url.to_s() unless url.nil?()
       
       return @links[url]
     end

data/lib/nhkore/search_scraper.rb

@@ -45,6 +45,16 @@ module NHKore
     # - https://www3.nhk.or.jp/news/easy/article/disaster_heat.html
     YASASHII_REGEX = /\A[^\.]+\.#{Regexp.quote(YASASHII_SITE)}.+\.html?/i
     
+    IGNORE_LINK_REGEX = %r{
+      /about\.html?            # https://www3.nhk.or.jp/news/easy/about.html
+      |/movieplayer\.html?      # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
+      |/audio\.html?            # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
+      |/news/easy/index\.html?  # http://www3.nhk.or.jp/news/easy/index.html
+      # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
+      # https://www3.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10012689671000&title=「鬼滅の刃」の映画が台湾でも始まって大勢の人が見に行く
+      |/enqform\.html?
+    }x
+    
     # Search Engines are strict, so trigger using the default HTTP header fields
     # with +header: {}+ and fetch/set the cookie using +eat_cookie: true+.
     def initialize(url,eat_cookie: true,header: {},**kargs)
@@ -57,10 +67,8 @@ module NHKore
       link = Util.unspace_web_str(link).downcase() unless cleaned
       
       return true if link.empty?()
-      return true if link =~ /\/about\.html?/ # https://www3.nhk.or.jp/news/easy/about.html
-      return true if link =~ /\/movieplayer\.html?/ # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
-      return true if link =~ /\/audio\.html?/ # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
-      return true if link =~ /\/news\/easy\/index\.html?/ # http://www3.nhk.or.jp/news/easy/index.html
+      
+      return true if IGNORE_LINK_REGEX.match?(link)
       
       return false
     end

data/lib/nhkore/sifter.rb

@@ -60,6 +60,40 @@ module NHKore
       @output = nil
     end
     
+    def build_header()
+      header = []
+      
+      header << 'Frequency' unless @ignores[:freq]
+      header << 'Word' unless @ignores[:word]
+      header << 'Kana' unless @ignores[:kana]
+      header << 'English' unless @ignores[:eng]
+      header << 'Definition' unless @ignores[:defn]
+      
+      return header
+    end
+    
+    def build_rows(words)
+      rows = []
+      
+      words.each() do |word|
+        rows << build_word_row(word)
+      end
+      
+      return rows
+    end
+    
+    def build_word_row(word)
+      row = []
+      
+      row << word.freq unless @ignores[:freq]
+      row << word.word unless @ignores[:word]
+      row << word.kana unless @ignores[:kana]
+      row << word.eng unless @ignores[:eng]
+      row << word.defn unless @ignores[:defn]
+      
+      return row
+    end
+    
     def filter?(article)
       return false if @filters.empty?()
       
@@ -151,26 +185,10 @@ module NHKore
       words = sift()
       
       @output = CSV.generate(headers: :first_row,write_headers: true) do |csv|
-        row = []
-        
-        row << 'Frequency' unless @ignores[:freq]
-        row << 'Word' unless @ignores[:word]
-        row << 'Kana' unless @ignores[:kana]
-        row << 'English' unless @ignores[:eng]
-        row << 'Definition' unless @ignores[:defn]
-        
-        csv << row
+        csv << build_header()
         
         words.each() do |word|
-          row = []
-          
-          row << word.freq unless @ignores[:freq]
-          row << word.word unless @ignores[:word]
-          row << word.kana unless @ignores[:kana]
-          row << word.eng unless @ignores[:eng]
-          row << word.defn unless @ignores[:defn]
-          
-          csv << row
+          csv << build_word_row(word)
         end
       end
       
@@ -232,7 +250,7 @@ module NHKore
         <h2>#{@caption}</h2>
         <table>
       EOH
-      #" # Fix for editor
+      #"
       
       # If have too few or too many '<col>', invalid HTML.
       @output << %Q{<col style="width:6em;">\n} unless @ignores[:freq]
@@ -242,20 +260,20 @@ module NHKore
       @output << "<col>\n" unless @ignores[:defn] # No width for defn, fills rest of page
       
       @output << '<tr>'
-      @output << '<th>Frequency</th>' unless @ignores[:freq]
-      @output << '<th>Word</th>' unless @ignores[:word]
-      @output << '<th>Kana</th>' unless @ignores[:kana]
-      @output << '<th>English</th>' unless @ignores[:eng]
-      @output << '<th>Definition</th>' unless @ignores[:defn]
+      
+      build_header().each() do |h|
+        @output << "<th>#{h}</th>"
+      end
+      
       @output << "</tr>\n"
       
       words.each() do |word|
         @output << '<tr>'
-        @output << "<td>#{Util.escape_html(word.freq.to_s())}</td>" unless @ignores[:freq]
-        @output << "<td>#{Util.escape_html(word.word.to_s())}</td>" unless @ignores[:word]
-        @output << "<td>#{Util.escape_html(word.kana.to_s())}</td>" unless @ignores[:kana]
-        @output << "<td>#{Util.escape_html(word.eng.to_s())}</td>" unless @ignores[:eng]
-        @output << "<td>#{Util.escape_html(word.defn.to_s())}</td>" unless @ignores[:defn]
+        
+        build_word_row(word).each() do |w|
+          @output << "<td>#{Util.escape_html(w.to_s())}</td>"
+        end
+        
         @output << "</tr>\n"
       end
       
@@ -264,31 +282,63 @@ module NHKore
         </body>
         </html>
       EOH
-      #/ # Fix for editor
+      #/
       
       return @output
     end
     
-    def put_yaml!()
+    def put_json!()
+      require 'json'
+      
       words = sift()
       
-      # Just blank out ignores.
-      if !@ignores.empty?()
-        words.each() do |word|
-          # word/kanji/kana do not have setters/mutators.
-          word.defn = nil if @ignores[:defn]
-          word.eng = nil if @ignores[:eng]
-          word.freq = nil if @ignores[:freq]
+      @output = ''.dup()
+      
+      @output << <<~EOJ
+        {
+        "caption": #{JSON.generate(@caption)},
+        "header": #{JSON.generate(build_header())},
+        "words": [
+      EOJ
+      
+      if !words.empty?()
+        0.upto(words.length - 2) do |i|
+          @output << "  #{JSON.generate(build_word_row(words[i]))},\n"
         end
+        
+        @output << "  #{JSON.generate(build_word_row(words[-1]))}\n"
       end
       
+      @output << "]\n}\n"
+      
+      return @output
+    end
+    
+    def put_yaml!()
+      require 'psychgus'
+      
+      words = sift()
+      
       yaml = {
         caption: @caption,
-        words: words
+        header: build_header(),
+        words: build_rows(words),
       }
       
+      header_styler = Class.new() do
+        include Psychgus::Styler
+        
+        def style_sequence(sniffer,node)
+          parent = sniffer.parent
+          
+          if !parent.nil?() && parent.node.respond_to?(:value) && parent.value == 'header'
+            node.style = Psychgus::SEQUENCE_FLOW
+          end
+        end
+      end
+      
       # Put each Word on one line (flow/inline style).
-      @output = Util.dump_yaml(yaml,flow_level: 4)
+      @output = Util.dump_yaml(yaml,flow_level: 4,stylers: header_styler.new())
       
       return @output
     end
@@ -306,7 +356,7 @@ module NHKore
       
       words = master_article.words.values()
       
-      words = words.sort() do |word1,word2|
+      words.sort!() do |word1,word2|
         # Order by freq DESC (most frequent words to top).
         i = (word2.freq <=> word1.freq)