nhkore 0.1.0 → 0.2.0

data/lib/nhkore/entry.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/defn'
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Entry
+    HYOUKI_SEP = '・'
+    
+    attr_reader :defns
+    attr_accessor :id
+    
+    def initialize()
+      super()
+      
+      @defns = []
+      @id = nil
+    end
+    
+    def build_defn()
+      defns = []
+      i = 0
+      
+      @defns.each() do |defn|
+        defns << "#{i += 1}）#{defn}" # Japanese parenthesis
+      end
+      
+      return defns.join("\n")
+    end
+    
+    def build_hyouki()
+      # Since Ruby v1.9, Hash preserves order.
+      # Ruby v2.7 doc for Set still says no guarantee of order, so don't use.
+      hyoukis = {}
+      
+      @defns.each() do |defn|
+        defn.hyoukis.each() do |hyouki|
+          hyouki = hyouki.chomp(HYOUKI_SEP)
+          
+          next if hyouki.empty?()
+          
+          hyoukis[hyouki] = true
+        end
+      end
+      
+      return hyoukis.keys.join(HYOUKI_SEP)
+    end
+    
+    def self.scrape(id,array,missingno: nil,url: nil)
+      entry = Entry.new()
+      
+      entry.id = Util.unspace_web_str(id.to_s()).downcase()
+      
+      return nil if entry.id.empty?()
+      
+      array.each() do |hash|
+        defn = Defn.scrape(hash,missingno: missingno,url: url)
+        entry.defns << defn unless defn.nil?()
+      end
+      
+      return nil if entry.defns.empty?()
+      return entry
+    end
+    
+    def to_s()
+      s = ''.dup()
+      
+      return s if @defns.empty?()
+      
+      hyouki = build_hyouki()
+      
+      s << "#{hyouki}\n" unless Util.empty_web_str?(hyouki)
+      s << build_defn()
+      
+      return s
+    end
+  end
+end

data/lib/nhkore/error.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Error < ::StandardError; end
+  
+  class CLIError < Error; end # @since 0.2.0
+  class ParseError < Error; end # @since 0.2.0
+  class ScrapeError < Error; end # @since 0.2.0
+  class ZipError < Error; end # @since 0.2.0
+end

data/lib/nhkore/fileable.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  module Fileable
+    def self.included(mod)
+      mod.extend ClassMethods
+    end
+    
+    def save_file(file,mode: 'wt',**kargs)
+      File.open(file,mode: mode,**kargs) do |file|
+        file.write(to_s())
+      end
+    end
+    
+    module ClassMethods
+      def load_file(file,mode: 'rt:BOM|UTF-8',**kargs)
+        data = File.read(file,mode: mode,**kargs)
+        
+        return load_data(data,file: file,**kargs)
+      end
+    end
+  end
+end

data/lib/nhkore/missingno.rb

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Missingno
+    attr_reader :kanas
+    attr_reader :kanjis
+    
+    # @param data [News,Article,Array<Word>]
+    def initialize(data)
+      super()
+      
+      @kanas = {}
+      @kanjis = {}
+      
+      # News?
+      if data.respond_to?(:articles)
+        add_news(data)
+      # Article?
+      elsif data.respond_to?(:words)
+        add_article(data)
+      else
+        add_words(data)
+      end
+    end
+    
+    def add_article(article)
+      add_words(article.words.values())
+    end
+    
+    def add_news(news)
+      news.articles.values.each() do |article|
+        add_article(article)
+      end
+    end
+    
+    def add_words(words)
+      words.each() do |word|
+        # We only want ones that are both filled in because
+        #   Word.scrape_ruby_tag() will raise an error if either is empty.
+        next if Util.empty_web_str?(word.kana) || Util.empty_web_str?(word.kanji)
+        
+        if !kanas.key?(word.kana)
+          kanas[word.kana] = word
+        end
+        
+        if !kanjis.key?(word.kanji)
+          kanjis[word.kanji] = word
+        end
+      end
+    end
+    
+    def kana_from_kanji(kanji)
+      word = @kanjis[kanji]
+      
+      return word.nil?() ? nil : word.kana
+    end
+    
+    def kanji_from_kana(kana)
+      word = @kanas[kana]
+      
+      return word.nil?() ? nil : word.kanji
+    end
+  end
+end

data/lib/nhkore/news.rb

@@ -0,0 +1,176 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/article'
+require 'nhkore/error'
+require 'nhkore/fileable'
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class News
+    include Fileable
+    
+    DEFAULT_DIR = Util::CORE_DIR
+    FAVORED_URL = /https\:/i
+    
+    attr_reader :articles
+    attr_reader :sha256s
+    
+    def initialize()
+      super()
+      
+      @articles = {}
+      @sha256s = {}
+    end
+    
+    def add_article(article,key: nil,overwrite: false)
+      key = article.url if key.nil?()
+      
+      if !overwrite
+        raise ArgumentError,"duplicate article[#{key}] in articles" if @articles.key?(key)
+        raise ArgumentError,"duplicate sha256[#{article.sha256}] in articles" if @sha256s.key?(article.sha256)
+      end
+      
+      @articles[key] = article
+      @sha256s[article.sha256] = article.url
+      
+      return self
+    end
+    
+    def self.build_file(filename)
+      return File.join(DEFAULT_DIR,filename)
+    end
+    
+    def encode_with(coder)
+      # Order matters.
+      # Don't output @sha256s.
+      
+      coder[:articles] = @articles
+    end
+    
+    def self.load_data(data,article_class: Article,file: nil,news_class: News,**kargs)
+      data = Util.load_yaml(data,file: file)
+      
+      articles = data[:articles]
+      
+      news = news_class.new()
+      
+      if !articles.nil?()
+        articles.each() do |key,hash|
+          key = key.to_s() # Change from a symbol
+          news.add_article(article_class.load_data(key,hash),key: key)
+        end
+      end
+      
+      return news
+    end
+    
+    def update_article(article,url)
+      # Favor https.
+      return if article.url =~ FAVORED_URL
+      return if url !~ FAVORED_URL
+      
+      @articles.delete(article.url)
+      @articles[url] = article
+      article.url = url
+    end
+    
+    def article(key)
+      return @articles[key]
+    end
+    
+    def article_with_sha256(sha256)
+      article = nil
+      
+      @articles.values().each() do |a|
+        if a.sha256 == sha256
+          article = a
+          
+          break
+        end
+      end
+      
+      return article
+    end
+    
+    def article?(key)
+      return @articles.key?(key)
+    end
+    
+    def sha256?(sha256)
+      return @sha256s.key?(sha256)
+    end
+    
+    def to_s()
+      # Put each Word on one line (flow/inline style).
+      return Util.dump_yaml(self,flow_level: 8)
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class FutsuuNews < News
+    DEFAULT_FILENAME = 'nhk_news_web_regular.yml'
+    DEFAULT_FILE = build_file(DEFAULT_FILENAME)
+    
+    def self.load_data(data,**kargs)
+      return News.load_data(data,article_class: Article,news_class: FutsuuNews,**kargs)
+    end
+    
+    def self.load_file(file=DEFAULT_FILE,**kargs)
+      return News.load_file(file,article_class: Article,news_class: FutsuuNews,**kargs)
+    end
+    
+    def save_file(file=DEFAULT_FILE,**kargs)
+      super(file,**kargs)
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class YasashiiNews < News
+    DEFAULT_FILENAME = 'nhk_news_web_easy.yml'
+    DEFAULT_FILE = build_file(DEFAULT_FILENAME)
+    
+    def self.load_data(data,**kargs)
+      return News.load_data(data,article_class: Article,news_class: YasashiiNews,**kargs)
+    end
+    
+    def self.load_file(file=DEFAULT_FILE,**kargs)
+      return News.load_file(file,article_class: Article,news_class: YasashiiNews,**kargs)
+    end
+    
+    def save_file(file=DEFAULT_FILE,**kargs)
+      super(file,**kargs)
+    end
+  end
+end

data/lib/nhkore/polisher.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/word'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Polisher
+    def begin_polish(str)
+      return str
+    end
+    
+    def polish(str)
+      str = begin_polish(str)
+      str = end_polish(str)
+      
+      return str
+    end
+    
+    def self.polish_any(obj,polishers)
+      return nil if obj.nil?()
+      
+      polishers = Array(polishers)
+      
+      return obj if polishers.empty?()
+      
+      if obj.is_a?(Word)
+        obj = Word.new(
+          kana: polish_any(obj.kana,polishers),
+          kanji: polish_any(obj.kanji,polishers),
+          word: obj
+        )
+      else # String
+        polishers.each() do |polisher|
+          obj = polisher.polish(obj)
+        end
+      end
+      
+      return obj
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BasicPolisher < Polisher
+    def end_polish(str)
+      # Keep Japanese dots in names:
+      # - Yunibaasaru・Sutajio・Japan
+      # Keep numbers next to kanji/kana, else the below kana won't make sense:
+      # - Word { kanji: ２０日, kana: はつか }
+      
+      str = str.gsub(/[^[[:alnum:]]・]/,'')
+      
+      # Numbers/dots by themselves (without kanji/kana) should be ignored (empty).
+      str = '' if str.gsub(/[[[:digit:]]・]+/,'').empty?()
+      
+      return str
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BestPolisher < BasicPolisher
+  end
+end

data/lib/nhkore/scraper.rb

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nokogiri'
+require 'open-uri'
+
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Scraper
+    # Copied from googler (https://github.com/jarun/googler).
+    USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
+    
+    attr_accessor :is_file
+    attr_accessor :str_or_io
+    attr_accessor :url
+    
+    alias_method :is_file?,:is_file
+    
+    # +max_redirects+ defaults to 3 for safety (infinite-loop attack).
+    # 
+    # All URL options: https://ruby-doc.org/stdlib-2.7.0/libdoc/open-uri/rdoc/OpenURI/OpenRead.html
+    # 
+    # Pass in +header: {}+ for the default HTTP header fields to be set.
+    # 
+    # @param redirect_rule [nil,:lenient,:strict]
+    def initialize(url,header: nil,is_file: false,max_redirects: 3,max_retries: 3,redirect_rule: :strict,str_or_io: nil,**kargs)
+      super()
+      
+      @is_file = is_file
+      @url = url
+      
+      if !header.nil?() && !is_file
+        # Some sites (Search Engines) hate scrapers, so need HTTP header fields.
+        # If this isn't enough, look at googler for more header fields to set:
+        # - https://github.com/jarun/googler
+        # If necessary, can use Faraday, HTTParty, or RestClient gem and
+        #   pass in to str_or_io.
+        
+        header['User-Agent'] = USER_AGENT unless header.key?('User-Agent')
+        
+        kargs.merge!(header) # header will overwrite duplicate kargs entries
+      end
+      
+      if str_or_io.nil?()
+        if is_file
+          # NHK's website tends to always use UTF-8.
+          @str_or_io = File.open(url,'rt:UTF-8',**kargs)
+        else
+          max_redirects = 10000 if max_redirects.nil?() || max_redirects < 0
+          
+          top_uri = URI(url)
+          top_domain = Util.domain(top_uri.host)
+          
+          begin
+            # Use URI.open() instead of (Kernel.)open() for safety (code-injection attack).
+            @str_or_io = URI.open(url,redirect: false,**kargs)
+            @url = url
+          rescue OpenURI::HTTPError => e
+            raise e.exception("HTTP error[#{e.to_s()}] at URL[#{url}]")
+          rescue OpenURI::HTTPRedirect => redirect
+            redirect_uri = redirect.uri
+            
+            if (max_redirects -= 1) < 0
+              raise redirect.exception("redirected to URL[#{redirect_uri}]: #{redirect}")
+            end
+            
+            case redirect_rule
+            when :lenient,:strict
+              if redirect_uri.scheme != top_uri.scheme
+                raise redirect.exception("redirect scheme[#{redirect_uri.scheme}] does not match original " +
+                  "scheme[#{top_uri.scheme}] at redirect URL[#{redirect_uri}]: #{redirect}")
+              end
+              
+              if redirect_rule == :strict
+                redirect_domain = Util.domain(redirect_uri.host)
+                
+                if redirect_domain != top_domain
+                  raise redirect.exception("redirect domain[#{redirect_domain}] does not match original " +
+                    "domain[#{top_domain}] at redirect URL[#{redirect_uri}]: #{redirect}")
+                end
+              end
+            end
+            
+            url = redirect_uri
+            
+            retry
+          rescue SocketError
+            raise if max_retries.nil?() || (max_retries -= 1) < 0
+            
+            retry
+          end
+        end
+      else
+        @str_or_io = str_or_io
+      end
+    end
+    
+    def html_doc()
+      return Nokogiri::HTML(@str_or_io)
+    end
+    
+    def join_url(relative_url)
+      # For a file, don't know what to do.
+      # It would be unsafe to return something else;
+      #   for example, it could return a lot of "../../../" to your root dir.
+      return nil if @is_file
+      
+      return URI::join(@url,relative_url)
+    end
+  end
+end