nhkore 0.3.1 → 0.3.6

data/lib/nhkore/scraper.rb

@@ -21,6 +21,7 @@
 #++
 
 
+require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
 
@@ -40,8 +41,8 @@ module NHKore
       'dnt' => '1',
     }
     
-    attr_accessor :eat_cookie
-    attr_accessor :is_file
+    attr_accessor? :eat_cookie
+    attr_accessor? :is_file
     attr_reader :kargs
     attr_accessor :max_redirects
     attr_accessor :max_retries
@@ -49,9 +50,6 @@ module NHKore
     attr_accessor :str_or_io
     attr_accessor :url
     
-    alias_method :eat_cookie?,:eat_cookie
-    alias_method :is_file?,:is_file
-    
     # +max_redirects+ defaults to 3 for safety (infinite-loop attack).
     # 
     # All URL options: https://ruby-doc.org/stdlib-2.7.0/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
@@ -82,7 +80,7 @@ module NHKore
       @max_retries = max_retries
       @redirect_rule = redirect_rule
       
-      open(url,str_or_io)
+      open(url,str_or_io,is_file: is_file)
     end
     
     def fetch_cookie(url)
@@ -119,14 +117,14 @@ module NHKore
       return URI::join(@url,relative_url)
     end
     
-    def open(url,str_or_io=nil)
+    def open(url,str_or_io=nil,is_file: @is_file)
+      @is_file = is_file
       @str_or_io = str_or_io
       @url = url
       
       if str_or_io.nil?()
         if @is_file
-          # NHK's website tends to always use UTF-8.
-          @str_or_io = File.open(url,'rt:UTF-8',**@kargs)
+          open_file(url)
         else
           fetch_cookie(url) if @eat_cookie
           open_url(url)
@@ -136,6 +134,16 @@ module NHKore
       return self
     end
     
+    def open_file(file)
+      @is_file = true
+      @url = file
+      
+      # NHK's website tends to always use UTF-8.
+      @str_or_io = File.open(file,'rt:UTF-8',**@kargs)
+      
+      return self
+    end
+    
     def open_url(url)
       max_redirects = (@max_redirects.nil?() || @max_redirects < 0) ? 10_000 : @max_redirects
       max_retries = (@max_retries.nil?() || @max_retries < 0) ? 10_000 : @max_retries
@@ -194,6 +202,10 @@ module NHKore
       return @str_or_io
     end
     
+    def reopen()
+      return open(@url)
+    end
+    
     def rss_doc()
       require 'rss'
       

data/lib/nhkore/search_link.rb

@@ -21,6 +21,7 @@
 #++
 
 
+require 'attr_bool'
 require 'time'
 
 require 'nhkore/fileable'
@@ -33,14 +34,12 @@ module NHKore
   # @since  0.2.0
   ###
   class SearchLink
-    attr_accessor :datetime
-    attr_accessor :futsuurl
-    attr_accessor :scraped
+    attr_reader :datetime
+    attr_reader :futsuurl
+    attr_accessor? :scraped
     attr_accessor :sha256
     attr_accessor :title
-    attr_accessor :url
-    
-    alias_method :scraped?,:scraped
+    attr_reader :url
     
     def initialize(url,scraped: false)
       super()
@@ -50,29 +49,27 @@ module NHKore
       @scraped = scraped
       @sha256 = sha256
       @title = nil
-      @url = url
+      self.url = url
     end
     
     def encode_with(coder)
       # Order matters.
       
-      coder[:url] = @url
+      coder[:url] = @url.nil?() ? nil : @url.to_s()
       coder[:scraped] = @scraped
-      coder[:datetime] = @datetime.nil?() ? @datetime : @datetime.iso8601()
+      coder[:datetime] = @datetime.nil?() ? nil : @datetime.iso8601()
       coder[:title] = @title
-      coder[:futsuurl] = @futsuurl
+      coder[:futsuurl] = @futsuurl.nil?() ? nil : @futsuurl.to_s()
       coder[:sha256] = @sha256
     end
     
     def self.load_data(key,hash)
-      datetime = hash[:datetime]
-      
       slink = SearchLink.new(
         hash[:url],
-        scraped: hash[:scraped]
+        scraped: hash[:scraped],
       )
       
-      slink.datetime = Util.empty_web_str?(datetime) ? nil : Time.iso8601(datetime)
+      slink.datetime = hash[:datetime]
       slink.futsuurl = hash[:futsuurl]
       slink.sha256 = hash[:sha256]
       slink.title = hash[:title]
@@ -83,13 +80,31 @@ module NHKore
     def update_from_article(article)
       # Don't update the url, as it may be different (e.g., http vs https).
       
-      @datetime = article.datetime if @datetime.nil?()
-      @futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
+      self.datetime = article.datetime if @datetime.nil?()
+      self.futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
       @scraped = true # If we have an article, it's been scraped
       @sha256 = article.sha256 if Util.empty_web_str?(@sha256)
       @title = article.title if Util.empty_web_str?(@title)
     end
     
+    def datetime=(value)
+      if value.is_a?(Time)
+        @datetime = value
+      else
+        @datetime = Util.empty_web_str?(value) ? nil : Time.iso8601(value)
+      end
+    end
+    
+    def futsuurl=(value)
+      # Don't store URI, store String.
+      @futsuurl = value.nil?() ? nil : value.to_s()
+    end
+    
+    def url=(value)
+      # Don't store URI, store String.
+      @url = value.nil?() ? nil : value.to_s()
+    end
+    
     def to_s(mini: false)
       s = ''.dup()
       
@@ -137,9 +152,11 @@ module NHKore
     end
     
     def add_link(link)
-      return self if @links.key?(link.url)
+      url = link.url.nil?() ? nil : link.url.to_s()
+      
+      return self if @links.key?(url)
       
-      @links[link.url] = link
+      @links[url] = link
       
       return self
     end
@@ -163,7 +180,7 @@ module NHKore
       
       if !links.nil?()
         links.each() do |key,hash|
-          key = key.to_s() # Change from a symbol
+          key = key.to_s() unless key.nil?()
           slinks.links[key] = SearchLink.load_data(key,hash)
         end
       end
@@ -173,6 +190,7 @@ module NHKore
     
     def [](url)
       url = url.url if url.respond_to?(:url)
+      url = url.to_s() unless url.nil?()
       
       return @links[url]
     end

data/lib/nhkore/search_scraper.rb

@@ -61,6 +61,7 @@ module NHKore
       return true if link =~ /\/movieplayer\.html?/ # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
       return true if link =~ /\/audio\.html?/ # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
       return true if link =~ /\/news\/easy\/index\.html?/ # http://www3.nhk.or.jp/news/easy/index.html
+      return true if link =~ /cgi2.*enqform/ # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
       
       return false
     end

data/lib/nhkore/sifter.rb

@@ -60,6 +60,40 @@ module NHKore
       @output = nil
     end
     
+    def build_header()
+      header = []
+      
+      header << 'Frequency' unless @ignores[:freq]
+      header << 'Word' unless @ignores[:word]
+      header << 'Kana' unless @ignores[:kana]
+      header << 'English' unless @ignores[:eng]
+      header << 'Definition' unless @ignores[:defn]
+      
+      return header
+    end
+    
+    def build_rows(words)
+      rows = []
+      
+      words.each() do |word|
+        rows << build_word_row(word)
+      end
+      
+      return rows
+    end
+    
+    def build_word_row(word)
+      row = []
+      
+      row << word.freq unless @ignores[:freq]
+      row << word.word unless @ignores[:word]
+      row << word.kana unless @ignores[:kana]
+      row << word.eng unless @ignores[:eng]
+      row << word.defn unless @ignores[:defn]
+      
+      return row
+    end
+    
     def filter?(article)
       return false if @filters.empty?()
       
@@ -93,24 +127,29 @@ module NHKore
       return false
     end
     
-    def filter_by_datetime(datetime_filter=nil,from_filter: nil,to_filter: nil)
+    def filter_by_datetime(datetime_filter=nil,from: nil,to: nil)
       if !datetime_filter.nil?()
-        # If out-of-bounds, just nil.
-        from_filter = datetime_filter[0]
-        to_filter = datetime_filter[1]
+        if datetime_filter.respond_to?(:'[]')
+          # If out-of-bounds, just nil.
+          from = datetime_filter[0] if from.nil?()
+          to = datetime_filter[1] if to.nil?()
+        else
+          from = datetime_filter if from.nil?()
+          to = datetime_filter if to.nil?()
+        end
       end
       
-      from_filter = to_filter if from_filter.nil?()
-      to_filter = from_filter if to_filter.nil?()
+      from = to if from.nil?()
+      to = from if to.nil?()
       
-      from_filter = Util.jst_time(from_filter) unless from_filter.nil?()
-      to_filter = Util.jst_time(to_filter) unless to_filter.nil?()
+      from = Util.jst_time(from) unless from.nil?()
+      to = Util.jst_time(to) unless to.nil?()
       
-      datetime_filter = [from_filter,to_filter]
+      datetime_filter = [from,to]
       
       return self if datetime_filter.flatten().compact().empty?()
       
-      @filters[:datetime] = {from: from_filter,to: to_filter}
+      @filters[:datetime] = {from: from,to: to}
       
       return self
     end
@@ -146,26 +185,10 @@ module NHKore
       words = sift()
       
       @output = CSV.generate(headers: :first_row,write_headers: true) do |csv|
-        row = []
-        
-        row << 'Frequency' unless @ignores[:freq]
-        row << 'Word' unless @ignores[:word]
-        row << 'Kana' unless @ignores[:kana]
-        row << 'English' unless @ignores[:eng]
-        row << 'Definition' unless @ignores[:defn]
-        
-        csv << row
+        csv << build_header()
         
         words.each() do |word|
-          row = []
-          
-          row << word.freq unless @ignores[:freq]
-          row << word.word unless @ignores[:word]
-          row << word.kana unless @ignores[:kana]
-          row << word.eng unless @ignores[:eng]
-          row << word.defn unless @ignores[:defn]
-          
-          csv << row
+          csv << build_word_row(word)
         end
       end
       
@@ -227,7 +250,7 @@ module NHKore
         <h2>#{@caption}</h2>
         <table>
       EOH
-      #" # Fix for editor
+      #"
       
       # If have too few or too many '<col>', invalid HTML.
       @output << %Q{<col style="width:6em;">\n} unless @ignores[:freq]
@@ -237,20 +260,20 @@ module NHKore
       @output << "<col>\n" unless @ignores[:defn] # No width for defn, fills rest of page
       
       @output << '<tr>'
-      @output << '<th>Frequency</th>' unless @ignores[:freq]
-      @output << '<th>Word</th>' unless @ignores[:word]
-      @output << '<th>Kana</th>' unless @ignores[:kana]
-      @output << '<th>English</th>' unless @ignores[:eng]
-      @output << '<th>Definition</th>' unless @ignores[:defn]
+      
+      build_header().each() do |h|
+        @output << "<th>#{h}</th>"
+      end
+      
       @output << "</tr>\n"
       
       words.each() do |word|
         @output << '<tr>'
-        @output << "<td>#{Util.escape_html(word.freq.to_s())}</td>" unless @ignores[:freq]
-        @output << "<td>#{Util.escape_html(word.word.to_s())}</td>" unless @ignores[:word]
-        @output << "<td>#{Util.escape_html(word.kana.to_s())}</td>" unless @ignores[:kana]
-        @output << "<td>#{Util.escape_html(word.eng.to_s())}</td>" unless @ignores[:eng]
-        @output << "<td>#{Util.escape_html(word.defn.to_s())}</td>" unless @ignores[:defn]
+        
+        build_word_row(word).each() do |w|
+          @output << "<td>#{Util.escape_html(w.to_s())}</td>"
+        end
+        
         @output << "</tr>\n"
       end
       
@@ -259,31 +282,63 @@ module NHKore
         </body>
         </html>
       EOH
-      #/ # Fix for editor
+      #/
       
       return @output
     end
     
-    def put_yaml!()
+    def put_json!()
+      require 'json'
+      
       words = sift()
       
-      # Just blank out ignores.
-      if !@ignores.empty?()
-        words.each() do |word|
-          # word/kanji/kana do not have setters/mutators.
-          word.defn = nil if @ignores[:defn]
-          word.eng = nil if @ignores[:eng]
-          word.freq = nil if @ignores[:freq]
+      @output = ''.dup()
+      
+      @output << <<~EOJ
+        {
+        "caption": #{JSON.generate(@caption)},
+        "header": #{JSON.generate(build_header())},
+        "words": [
+      EOJ
+      
+      if !words.empty?()
+        0.upto(words.length - 2) do |i|
+          @output << "  #{JSON.generate(build_word_row(words[i]))},\n"
         end
+        
+        @output << "  #{JSON.generate(build_word_row(words[-1]))}\n"
       end
       
+      @output << "]\n}\n"
+      
+      return @output
+    end
+    
+    def put_yaml!()
+      require 'psychgus'
+      
+      words = sift()
+      
       yaml = {
         caption: @caption,
-        words: words
+        header: build_header(),
+        words: build_rows(words),
       }
       
+      header_styler = Class.new() do
+        include Psychgus::Styler
+        
+        def style_sequence(sniffer,node)
+          parent = sniffer.parent
+          
+          if !parent.nil?() && parent.node.respond_to?(:value) && parent.value == 'header'
+            node.style = Psychgus::SEQUENCE_FLOW
+          end
+        end
+      end
+      
       # Put each Word on one line (flow/inline style).
-      @output = Util.dump_yaml(yaml,flow_level: 4)
+      @output = Util.dump_yaml(yaml,flow_level: 4,stylers: header_styler.new())
       
       return @output
     end
@@ -301,7 +356,7 @@ module NHKore
       
       words = master_article.words.values()
       
-      words = words.sort() do |word1,word2|
+      words.sort!() do |word1,word2|
         # Order by freq DESC (most frequent words to top).
         i = (word2.freq <=> word1.freq)
         

data/lib/nhkore/util.rb

@@ -22,8 +22,7 @@
 
 
 require 'cgi'
-require 'psychgus'
-require 'public_suffix'
+require 'set'
 require 'time'
 require 'uri'
 
@@ -68,21 +67,28 @@ module NHKore
     end
     
     def self.domain(host,clean: true)
+      require 'public_suffix'
+      
       domain = PublicSuffix.domain(host)
       domain = unspace_web_str(domain).downcase() if !domain.nil?() && clean
       
       return domain
     end
     
-    def self.dump_yaml(obj,flow_level: 8)
+    def self.dump_yaml(obj,flow_level: 8,stylers: nil)
+      require 'psychgus'
+      
+      stylers = Array(stylers)
+      
       return Psychgus.dump(obj,
         deref_aliases: true, # Dereference aliases for load_yaml()
+        header: true, # %YAML [version]
         line_width: 10000, # Try not to wrap; ichiman!
         stylers: [
           Psychgus::FlowStyler.new(flow_level), # Put extra details on one line (flow/inline style)
           Psychgus::NoSymStyler.new(cap: false), # Remove symbols, don't capitalize
           Psychgus::NoTagStyler.new(), # Remove class names (tags)
-        ],
+        ].concat(stylers),
       )
     end
     
@@ -102,23 +108,6 @@ module NHKore
       return !str.match?(/[\/\\]/)
     end
     
-    def self.guess_year(year)
-      if year < 100
-        # 2021 -> 2000.
-        millennium = JST_YEAR / 100 * 100
-        
-        # If year <= (2021 -> 21), assume this century.
-        if year <= (JST_YEAR % 100)
-          year = millennium + year
-        else
-          # Assume previous century (2000 -> 1900).
-          year = (millennium - 100) + year
-        end
-      end
-      
-      return year
-    end
-    
     def self.hiragana?(str)
       return HIRAGANA_REGEX =~ str
     end
@@ -142,6 +131,8 @@ module NHKore
     end
     
     def self.load_yaml(data,file: nil,**kargs)
+      require 'psychgus'
+      
       return Psych.safe_load(data,
         aliases: false,
         filename: file,