nhkore 0.1.0 → 0.2.0

data/lib/nhkore/search_link.rb

@@ -0,0 +1,188 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'time'
+
+require 'nhkore/fileable'
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class SearchLink
+    attr_accessor :datetime
+    attr_accessor :futsuurl
+    attr_accessor :scraped
+    attr_accessor :sha256
+    attr_accessor :title
+    attr_accessor :url
+    
+    alias_method :scraped?,:scraped
+    
+    def initialize(url,scraped: false)
+      super()
+      
+      @datetime = nil
+      @futsuurl = nil
+      @scraped = scraped
+      @sha256 = sha256
+      @title = nil
+      @url = url
+    end
+    
+    def encode_with(coder)
+      # Order matters.
+      
+      coder[:url] = @url
+      coder[:scraped] = @scraped
+      coder[:datetime] = @datetime.nil?() ? @datetime : @datetime.iso8601()
+      coder[:title] = @title
+      coder[:futsuurl] = @futsuurl
+      coder[:sha256] = @sha256
+    end
+    
+    def self.load_data(key,hash)
+      datetime = hash[:datetime]
+      
+      slink = SearchLink.new(
+        hash[:url],
+        scraped: hash[:scraped]
+      )
+      
+      slink.datetime = Util.empty_web_str?(datetime) ? nil : Time.iso8601(datetime)
+      slink.futsuurl = hash[:futsuurl]
+      slink.sha256 = hash[:sha256]
+      slink.title = hash[:title]
+      
+      return slink
+    end
+    
+    def update_from_article(article)
+      # Don't update the url, as it may be different (e.g., http vs https).
+      
+      @datetime = article.datetime if @datetime.nil?()
+      @futsuurl = article.futsuurl if Util.empty_web_str?(@futsuurl)
+      @scraped = true # If we have an article, it's been scraped
+      @sha256 = article.sha256 if Util.empty_web_str?(@sha256)
+      @title = article.title if Util.empty_web_str?(@title)
+    end
+    
+    def to_s(mini: false)
+      s = ''.dup()
+      
+      s << "'#{@url}': "
+      
+      if mini
+        s << "{ scraped? #{@scraped ? 'yes' : 'NO'} }"
+      else
+        s << "\n  scraped?  #{@scraped ? 'yes' : 'NO'}"
+        s << "\n  datetime: '#{@datetime}'"
+        s << "\n  title:    '#{@title}'"
+        s << "\n  futsuurl: '#{@futsuurl}'"
+        s << "\n  sha256:   '#{@sha256}'"
+      end
+      
+      return s
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class SearchLinks
+    include Fileable
+    
+    DEFAULT_DIR = Util::CORE_DIR
+    
+    DEFAULT_BING_FUTSUU_FILENAME = 'bing_nhk_news_web_regular.yml'
+    DEFAULT_BING_YASASHII_FILENAME = 'bing_nhk_news_web_easy.yml'
+    
+    def self.build_file(filename)
+      return File.join(DEFAULT_DIR,filename)
+    end
+    
+    DEFAULT_BING_FUTSUU_FILE = build_file(DEFAULT_BING_FUTSUU_FILENAME)
+    DEFAULT_BING_YASASHII_FILE = build_file(DEFAULT_BING_YASASHII_FILENAME)
+    
+    attr_reader :links
+    
+    def initialize()
+      super()
+      
+      @links = {}
+    end
+    
+    def add_link(link)
+      return self if @links.key?(link.url)
+      
+      @links[link.url] = link
+      
+      return self
+    end
+    
+    def each(&block)
+      return @links.each(&block)
+    end
+    
+    def encode_with(coder)
+      # Order matters.
+      
+      coder[:links] = @links
+    end
+    
+    def self.load_data(data,file: nil,**kargs)
+      data = Util.load_yaml(data,file: file)
+      
+      links = data[:links]
+      
+      slinks = SearchLinks.new()
+      
+      if !links.nil?()
+        links.each() do |key,hash|
+          key = key.to_s() # Change from a symbol
+          slinks.links[key] = SearchLink.load_data(key,hash)
+        end
+      end
+      
+      return slinks
+    end
+    
+    def [](url)
+      url = url.url if url.respond_to?(:url)
+      
+      return @links[url]
+    end
+    
+    def length()
+      return @links.length
+    end
+    
+    def to_s()
+      return Util.dump_yaml(self)
+    end
+  end
+end

data/lib/nhkore/search_scraper.rb

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'uri'
+
+require 'nhkore/error'
+require 'nhkore/scraper'
+require 'nhkore/search_link'
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class SearchScraper < Scraper
+    DEFAULT_RESULT_COUNT = 100
+    FUTSUU_SITE = 'nhk.or.jp/news/html/'
+    YASASHII_SITE = 'nhk.or.jp/news/easy/'
+    
+    # https://www3.nhk.or.jp/news/html/20200220/k10012294001000.html
+    FUTSUU_REGEX = /\A[^\.]+\.#{Regexp.quote(FUTSUU_SITE)}.+\.html?/i
+    # https://www3.nhk.or.jp/news/easy/k10012294001000/k10012294001000.html
+    # - https://www3.nhk.or.jp/news/easy/article/disaster_heat.html
+    YASASHII_REGEX = /\A[^\.]+\.#{Regexp.quote(YASASHII_SITE)}.+\.html?/i
+    
+    # Pass in +header: {}+ to trigger using the default HTTP header fields.
+    def initialize(url,header: {},**kargs)
+      super(url,header: header,**kargs)
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BingScraper < SearchScraper
+    attr_reader :regex
+    attr_reader :site
+    
+    def initialize(site,regex: nil,url: nil,**kargs)
+      case site
+      when :futsuu
+        regex = FUTSUU_REGEX if regex.nil?()
+        site = FUTSUU_SITE
+      when :yasashii
+        regex = YASASHII_REGEX if regex.nil?()
+        site = YASASHII_SITE
+      else
+        site = Util.strip_web_str(site.to_s())
+        regex = /#{Regexp.quote(site)}/ if regex.nil?()
+      end
+      
+      raise ArgumentError,"empty regex[#{regex}]" if regex.nil?()
+      raise ArgumentError,"empty site[#{site}]" if site.empty?()
+      
+      @regex = regex
+      @site = site
+      url = self.class.build_url(site,**kargs) if url.nil?()
+      
+      # Delete class-specific args (don't pass to Open-URI).
+      kargs.delete(:count)
+      
+      super(url,**kargs)
+    end
+    
+    def self.build_url(site,count: DEFAULT_RESULT_COUNT,**kargs)
+      url = ''.dup()
+      
+      url << 'https://www.bing.com/search?'
+      url << URI.encode_www_form(
+        q: "site:#{site}",
+        count: count
+      )
+      
+      return url
+    end
+    
+    def scrape(links,page=NextPage.new())
+      doc = html_doc()
+      next_page = NextPage.new()
+      
+      anchors = doc.css('a')
+      
+      return next_page if anchors.length < 1
+      
+      anchors.each() do |anchor|
+        href = anchor['href'].to_s()
+        href = Util.unspace_web_str(href).downcase()
+        
+        next if href.empty?()
+        next if href =~ /\/about\.html?/ # https://www3.nhk.or.jp/news/easy/about.html
+        next if href =~ /\/movieplayer\.html?/ # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
+        next if href =~ /\/audio\.html?/ # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
+        
+        if (md = href.match(/first\=(\d+)/))
+          count = md[1].to_i()
+          
+          if count > page.count && (next_page.count < 0 || count < next_page.count)
+            next_page.count = count
+            next_page.url = join_url(href)
+          end
+        elsif href =~ regex
+          links.add_link(SearchLink.new(href))
+        end
+      end
+      
+      return next_page
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class NextPage
+    attr_accessor :count
+    attr_accessor :url
+    
+    def initialize()
+      super()
+      
+      @count = -1
+      @url = nil
+    end
+    
+    def empty?()
+      return @url.nil?() || @count < 0
+    end
+  end
+end

data/lib/nhkore/sifter.rb

@@ -0,0 +1,339 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'csv'
+
+require 'nhkore/article'
+require 'nhkore/fileable'
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Sifter
+    include Fileable
+    
+    DEFAULT_DIR = Util::CORE_DIR
+    
+    DEFAULT_FUTSUU_FILENAME = 'sift_nhk_news_web_regular'
+    DEFAULT_YASASHII_FILENAME = 'sift_nhk_news_web_easy'
+    
+    def self.build_file(filename)
+      return File.join(DEFAULT_DIR,filename)
+    end
+    
+    DEFAULT_FUTSUU_FILE = build_file(DEFAULT_FUTSUU_FILENAME)
+    DEFAULT_YASASHII_FILE = build_file(DEFAULT_YASASHII_FILENAME)
+    
+    attr_accessor :articles
+    attr_accessor :caption
+    attr_accessor :filters
+    attr_accessor :ignores
+    attr_accessor :output
+    
+    def initialize(news)
+      @articles = news.articles.values.dup()
+      @caption = nil
+      @filters = {}
+      @ignores = {}
+      @output = nil
+    end
+    
+    def filter?(article)
+      return false if @filters.empty?()
+      
+      datetime_filter = @filters[:datetime]
+      title_filter = @filters[:title]
+      url_filter = @filters[:url]
+      
+      if !datetime_filter.nil?()
+        datetime = article.datetime
+        
+        return true if datetime.nil?() ||
+          datetime < datetime_filter[:from] || datetime > datetime_filter[:to]
+      end
+      
+      if !title_filter.nil?()
+        title = article.title.to_s()
+        title = Util.unspace_web_str(title) if title_filter[:unspace]
+        title = title.downcase() if title_filter[:uncase]
+        
+        return true unless title.include?(title_filter[:filter])
+      end
+      
+      if !url_filter.nil?()
+        url = article.url.to_s()
+        url = Util.unspace_web_str(url) if url_filter[:unspace]
+        url = url.downcase() if url_filter[:uncase]
+        
+        return true unless url.include?(url_filter[:filter])
+      end
+      
+      return false
+    end
+    
+    def filter_by_datetime(datetime_filter=nil,from_filter: nil,to_filter: nil)
+      if !datetime_filter.nil?()
+        # If out-of-bounds, just nil.
+        from_filter = datetime_filter[0]
+        to_filter = datetime_filter[1]
+      end
+      
+      from_filter = to_filter if from_filter.nil?()
+      to_filter = from_filter if to_filter.nil?()
+      
+      from_filter = Util.jst_time(from_filter) unless from_filter.nil?()
+      to_filter = Util.jst_time(to_filter) unless to_filter.nil?()
+      
+      datetime_filter = [from_filter,to_filter]
+      
+      return self if datetime_filter.flatten().compact().empty?()
+      
+      @filters[:datetime] = {from: from_filter,to: to_filter}
+      
+      return self
+    end
+    
+    def filter_by_title(title_filter,uncase: true,unspace: true)
+      title_filter = Util.unspace_web_str(title_filter) if unspace
+      title_filter = title_filter.downcase() if uncase
+      
+      @filters[:title] = {filter: title_filter,uncase: uncase,unspace: unspace}
+      
+      return self
+    end
+    
+    def filter_by_url(url_filter,uncase: true,unspace: true)
+      url_filter = Util.unspace_web_str(url_filter) if unspace
+      url_filter = url_filter.downcase() if uncase
+      
+      @filters[:url] = {filter: url_filter,uncase: uncase,unspace: unspace}
+      
+      return self
+    end
+    
+    def ignore(key)
+      @ignores[key] = true
+      
+      return self
+    end
+    
+    # This does not output {caption}.
+    def put_csv!()
+      words = sift()
+      
+      @output = CSV.generate(headers: :first_row,write_headers: true) do |csv|
+        row = []
+        
+        row << 'Frequency' unless @ignores[:freq]
+        row << 'Word' unless @ignores[:word]
+        row << 'Kana' unless @ignores[:kana]
+        row << 'English' unless @ignores[:eng]
+        row << 'Definition' unless @ignores[:defn]
+        
+        csv << row
+        
+        words.each() do |word|
+          row = []
+          
+          row << word.freq unless @ignores[:freq]
+          row << word.word unless @ignores[:word]
+          row << word.kana unless @ignores[:kana]
+          row << word.eng unless @ignores[:eng]
+          row << word.defn unless @ignores[:defn]
+          
+          csv << row
+        end
+      end
+      
+      return @output
+    end
+    
+    def put_html!()
+      words = sift()
+      
+      @output = ''.dup()
+      
+      @output << <<~EOH
+        <!DOCTYPE html>
+        <html lang="ja">
+        <head>
+        <meta charset="utf-8">
+        <title>NHKore</title>
+        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Noto+Serif+JP&amp;display=fallback">
+        <style>
+        body {
+          background-color: #FCFBF9;
+          color: #333333;
+          font-family: 'Noto Serif JP',Verdana,sans-serif;
+        }
+        h1 {
+          color: #737373;
+        }
+        table {
+          border-collapse: collapse;
+          table-layout: fixed;
+          width: 100%;
+        }
+        tr:nth-child(even) {
+          background-color: #A5C7ED;
+        }
+        tr:hover {
+          background-color: #FFDDCA;
+        }
+        td,th {
+          border: 1px solid #333333;
+          padding: 8px;
+          text-align: left;
+        }
+        th {
+          background-color: #082A8E;
+          color: #FCFBF9;
+        }
+        td {
+          vertical-align: top;
+        }
+        td:nth-child(1) {
+          padding-right: 1em;
+          text-align: right;
+        }
+        </style>
+        </head>
+        <body>
+        <h1>NHKore</h1>
+        <h2>#{@caption}</h2>
+        <table>
+      EOH
+      #" # Fix for editor
+      
+      # If have too few or too many '<col>', invalid HTML.
+      @output << %Q{<col style="width:6em;">\n} unless @ignores[:freq]
+      @output << %Q{<col style="width:17em;">\n} unless @ignores[:word]
+      @output << %Q{<col style="width:17em;">\n} unless @ignores[:kana]
+      @output << %Q{<col style="width:5em;">\n} unless @ignores[:eng]
+      @output << "<col>\n" unless @ignores[:defn] # No width for defn, fills rest of page
+      
+      @output << '<tr>'
+      @output << '<th>Frequency</th>' unless @ignores[:freq]
+      @output << '<th>Word</th>' unless @ignores[:word]
+      @output << '<th>Kana</th>' unless @ignores[:kana]
+      @output << '<th>English</th>' unless @ignores[:eng]
+      @output << '<th>Definition</th>' unless @ignores[:defn]
+      @output << "</tr>\n"
+      
+      words.each() do |word|
+        @output << '<tr>'
+        @output << "<td>#{Util.escape_html(word.freq.to_s())}</td>" unless @ignores[:freq]
+        @output << "<td>#{Util.escape_html(word.word.to_s())}</td>" unless @ignores[:word]
+        @output << "<td>#{Util.escape_html(word.kana.to_s())}</td>" unless @ignores[:kana]
+        @output << "<td>#{Util.escape_html(word.eng.to_s())}</td>" unless @ignores[:eng]
+        @output << "<td>#{Util.escape_html(word.defn.to_s())}</td>" unless @ignores[:defn]
+        @output << "</tr>\n"
+      end
+      
+      @output << <<~EOH
+        </table>
+        </body>
+        </html>
+      EOH
+      #/ # Fix for editor
+      
+      return @output
+    end
+    
+    def put_yaml!()
+      words = sift()
+      
+      # Just blank out ignores.
+      if !@ignores.empty?()
+        words.each() do |word|
+          # word/kanji/kana do not have setters/mutators.
+          word.defn = nil if @ignores[:defn]
+          word.eng = nil if @ignores[:eng]
+          word.freq = nil if @ignores[:freq]
+        end
+      end
+      
+      yaml = {
+        caption: @caption,
+        words: words
+      }
+      
+      # Put each Word on one line (flow/inline style).
+      @output = Util.dump_yaml(yaml,flow_level: 4)
+      
+      return @output
+    end
+    
+    def sift()
+      master_article = Article.new()
+      
+      @articles.each() do |article|
+        next if filter?(article)
+        
+        article.words.values().each() do |word|
+          master_article.add_word(word,use_freq: true)
+        end
+      end
+      
+      words = master_article.words.values()
+      
+      words = words.sort() do |word1,word2|
+        # Order by freq DESC (most frequent words to top).
+        i = (word2.freq <=> word1.freq)
+        
+        # Order by !defn.empty, word ASC, !kana.empty, kana ASC, defn.len DESC, defn ASC.
+        i = compare_empty_str(word1.defn,word2.defn) if i == 0 # Favor words that have definitions
+        i = (word1.word.to_s() <=> word2.word.to_s()) if i == 0
+        i = compare_empty_str(word1.kana,word2.kana) if i == 0 # Favor words that have kana
+        i = (word1.kana.to_s() <=> word2.kana.to_s()) if i == 0
+        i = (word2.defn.to_s().length <=> word1.defn.to_s().length) if i == 0 # Favor longer definitions
+        i = (word1.defn.to_s() <=> word2.defn.to_s()) if i == 0
+        
+        i
+      end
+      
+      return words
+    end
+    
+    def compare_empty_str(str1,str2)
+      has_str1 = !Util.empty_web_str?(str1)
+      has_str2 = !Util.empty_web_str?(str2)
+      
+      if has_str1 && !has_str2
+        return -1 # Bubble word1 to top
+      elsif !has_str1 && has_str2
+        return 1 # Bubble word2 to top
+      end
+      
+      return 0 # Further comparison needed
+    end
+    
+    def to_s()
+      return @output.to_s()
+    end
+  end
+end