nhkore 0.1.0 → 0.2.0

data/lib/nhkore/splitter.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'bimyou_segmenter'
+require 'tiny_segmenter'
+
+require 'nhkore/util'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Splitter
+    def begin_split(str)
+      return str
+    end
+    
+    def split(str)
+      str = begin_split(str)
+      str = end_split(str)
+      
+      return str
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BasicSplitter < Splitter
+    def end_split(str)
+      return str.split(Util::NORMALIZE_STR_REGEX)
+    end
+  end
+  
+  ###
+  # @since  0.2.0
+  ###
+  class BimyouSplitter < Splitter
+    def end_split(str)
+      return BimyouSegmenter.segment(str,symbol: false,white_space: false)
+    end
+  end
+  
+  ###
+  # @since  0.2.0
+  ###
+  class TinySplitter < Splitter
+    attr_accessor :tiny
+    
+    def initialize(*)
+      super
+      
+      @tiny = TinySegmenter.new()
+    end
+    
+    def end_split(str)
+      return @tiny.segment(str,ignore_punctuation: true)
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BestSplitter < BimyouSplitter
+  end
+end

data/lib/nhkore/util.rb

@@ -0,0 +1,190 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'cgi'
+require 'psychgus'
+require 'public_suffix'
+require 'time'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  module Util
+    CORE_DIR = 'core'
+    WEB_DIR = 'web'
+    
+    JST_OFFSET = '+09:00' # Japan Standard Time (JST) time zone offset from UTC
+    JST_OFFSET_HOUR = 9
+    JST_OFFSET_MIN = 0
+    
+    HIRAGANA_REGEX = /\p{Hiragana}/
+    JPN_SPACE = "\u3000" # Must be double-quoted for escape chars
+    KANA_REGEX = /\p{Hiragana}|\p{Katakana}/
+    KANJI_REGEX = /\p{Han}/ # Han probably stands for Hanzi?
+    KATAKANA_REGEX = /\p{Katakana}/
+    NORMALIZE_STR_REGEX = /[^[[:alpha:]]]+/
+    STRIP_WEB_STR_REGEX = /(\A[[:space:]]+)|([[:space:]]+\z)/
+    WEB_SPACES_REGEX = /[[:space:]]+/
+    
+    def self.jst_now()
+      return Time.now().getlocal(JST_OFFSET)
+    end
+    
+    JST_YEAR = jst_now().year
+    MAX_SANE_YEAR = JST_YEAR + 1 # +1 Justin Case for time zone differences at the end of the year
+    
+    # NHK was founded in 1924/25.
+    # - https://www.nhk.or.jp/bunken/english/about/history.html
+    # - https://en.wikipedia.org/wiki/NHK
+    # However, when was the website first created?
+    MIN_SANE_YEAR = 1924
+    
+    def self.dir_str?(str)
+      # File.join() will add the appropriate slash.
+      return File.join(str,'') == str
+    end
+    
+    def self.domain(host,clean: true)
+      domain = PublicSuffix.domain(host)
+      domain = unspace_web_str(domain).downcase() if clean
+      
+      return domain
+    end
+    
+    def self.dump_yaml(obj,flow_level: 8)
+      return Psychgus.dump(obj,
+        deref_aliases: true, # Dereference aliases for load_yaml()
+        line_width: 10000, # Try not to wrap; ichiman!
+        stylers: [
+          Psychgus::FlowStyler.new(flow_level), # Put extra details on one line (flow/inline style)
+          Psychgus::NoSymStyler.new(cap: false), # Remove symbols, don't capitalize
+          Psychgus::NoTagStyler.new(), # Remove class names (tags)
+        ],
+      )
+    end
+    
+    def self.empty_web_str?(str)
+      return str.nil?() || strip_web_str(str).empty?()
+    end
+    
+    def self.escape_html(str)
+      str = CGI.escapeHTML(str)
+      str = str.gsub("\n",'<br>')
+      
+      return str
+    end
+    
+    def self.filename_str?(str)
+      return File.basename(str) == str
+    end
+    
+    def self.guess_year(year)
+      if year < 100
+        # 2021 -> 2000.
+        millennium = JST_YEAR / 100 * 100
+        
+        # If year <= (2021 -> 21), assume this century.
+        if year <= (JST_YEAR % 100)
+          year = millennium + year
+        else
+          # Assume previous century (2000 -> 1900).
+          year = (millennium - 100) + year
+        end
+      end
+      
+      return year
+    end
+    
+    def self.hiragana?(str)
+      return HIRAGANA_REGEX =~ str
+    end
+    
+    # This doesn't modify the hour/minute according to {JST_OFFSET},
+    # but instead, it just drops {JST_OFFSET} into it without adjusting it.
+    def self.jst_time(time)
+      return Time.new(time.year,time.month,time.day,time.hour,time.min,time.sec,JST_OFFSET)
+    end
+    
+    def self.kana?(str)
+      return KANA_REGEX =~ str
+    end
+    
+    def self.kanji?(str)
+      return KANJI_REGEX =~ str
+    end
+    
+    def self.katakana?(str)
+      return KATAKANA_REGEX =~ str
+    end
+    
+    def self.load_yaml(data,file: nil,**kargs)
+      return Psych.safe_load(data,
+        aliases: false,
+        filename: file,
+        #freeze: true, # Not in this current version of Psych
+        permitted_classes: [Symbol],
+        symbolize_names: true,
+        **kargs,
+      )
+    end
+    
+    def self.normalize_str(str)
+      return str.gsub(NORMALIZE_STR_REGEX,'')
+    end
+    
+    def self.reduce_jpn_space(str)
+      # Do not strip; use a Japanese space
+      return str.gsub(WEB_SPACES_REGEX,JPN_SPACE)
+    end
+    
+    def self.reduce_space(str)
+      return str.gsub(WEB_SPACES_REGEX,' ')
+    end
+    
+    def self.sane_year?(year)
+      return year >= MIN_SANE_YEAR && year <= MAX_SANE_YEAR
+    end
+    
+    # String's normal strip() method doesn't work with special Unicode/HTML white space.
+    def self.strip_web_str(str)
+      # After testing with Benchmark, this is slower than one regex.
+      #str = str.gsub(/\A[[:space:]]+/,'')
+      #str = str.gsub(/[[:space:]]+\z/,'')
+      
+      str = str.gsub(STRIP_WEB_STR_REGEX,'')
+      
+      return str
+    end
+    
+    def self.unspace_web_str(str)
+      return str.gsub(WEB_SPACES_REGEX,'')
+    end
+    
+    def self.warn(msg,uplevel: 1)
+      Kernel.warn(msg,uplevel: uplevel)
+    end
+  end
+end

data/lib/nhkore/variator.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'japanese_deinflector'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Variator
+    def begin_variate(str)
+      return str
+    end
+    
+    def variate(str)
+      str = begin_variate(str)
+      str = end_variate(str)
+      
+      return str
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BasicVariator < Variator
+    def end_variate(str)
+      return [] # No variations; don't return nil
+    end
+  end
+  
+  ###
+  # Guesses a word's dictionary/plain form (辞書形).
+  # 
+  # It doesn't work very well,but better than nothing...
+  # 
+  # @since  0.2.0
+  ###
+  class DictFormVariator < Variator
+    attr_accessor :deinflector
+    
+    def initialize(*)
+      super
+      
+      @deinflector = JapaneseDeinflector.new()
+    end
+    
+    def end_variate(str)
+      guess = @deinflector.deinflect(str)
+      
+      return [] if guess.length < 1
+      return [] if (guess = guess[0])[:weight] < 0.5
+      
+      return [guess[:word]]
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BestVariator < DictFormVariator
+  end
+end

data/lib/nhkore/version.rb

@@ -22,5 +22,5 @@
 
 
 module NHKore
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/nhkore/word.rb

@@ -21,57 +21,174 @@
 #++
 
 
+require 'nokogiri'
+
+require 'nhkore/error'
+require 'nhkore/util'
+
+
 module NHKore
   ###
   # @author Jonathan Bradley Whited (@esotericpig)
   # @since  0.1.0
   ###
   class Word
+    attr_accessor :defn
+    attr_accessor :eng
     attr_accessor :freq
     attr_reader :kana
-    attr_reader :word
+    attr_reader :kanji
+    attr_reader :key
     
-    def initialize(word=nil,freq: 1,kana: nil,**kargs)
+    def initialize(defn: nil,eng: nil,freq: 1,kana: nil,kanji: nil,unknown: nil,word: nil,**kargs)
       super()
       
+      if !word.nil?()
+        defn = word.defn if defn.nil?()
+        eng = word.eng if eng.nil?()
+        freq = word.freq if freq.nil?()
+        kana = word.kana if kana.nil?()
+        kanji = word.kanji if kanji.nil?()
+      end
+      
+      raise ArgumentError,"freq[#{freq}] cannot be < 1" if freq < 1
+      
+      if !unknown.nil?()
+        # kanji?() only tests if it contains kanji, so don't use kana?().
+        if Util.kanji?(unknown)
+          raise ArgumentError,"unknown[#{unknown}] will overwrite kanji[#{kanji}]" unless Util.empty_web_str?(kanji)
+          
+          kanji = unknown
+        else
+          raise ArgumentError,"unknown[#{unknown}] will overwrite kana[#{kana}]" unless Util.empty_web_str?(kana)
+          
+          kana = unknown
+        end
+      end
+      
+      kana = nil if Util.empty_web_str?(kana)
+      kanji = nil if Util.empty_web_str?(kanji)
+      
+      raise ArgumentError,'kanji and kana cannot both be empty' if kana.nil?() && kanji.nil?()
+      
+      @defn = defn
+      @eng = eng
       @freq = freq
       @kana = kana
-      @word = word.nil?() ? kana : word
-      
-      raise ArgumentError,'word and kana cannot both be nil; one must be specified' if @word.nil?()
+      @kanji = kanji
+      @key = "#{kanji}=#{kana}" # nil.to_s() is ''
     end
     
     def encode_with(coder)
-      # Ignore @word because it will be the key in the YAML/Hash.
+      # Ignore @key because it will be the key in the YAML/Hash.
       # Order matters.
+      
+      coder[:kanji] = @kanji
       coder[:kana] = @kana
       coder[:freq] = @freq
+      coder[:defn] = @defn
+      coder[:eng] = @eng
     end
     
-    def self.load_hash(key,hash)
-      word = Word.new(key,kana: hash[:kana])
+    def self.load_data(key,hash)
+      key = key.to_s() # Change from a symbol
+      
+      word = Word.new(
+        defn: hash[:defn],
+        eng: hash[:eng],
+        kana: hash[:kana],
+        kanji: hash[:kanji]
+      )
+      
+      if key != word.key
+        raise ArgumentError,"the key from the hash[#{key}] does not match the generated key[#{word.key}]"
+      end
+      
+      freq = hash[:freq].to_i() # nil.to_i() is 0
+      word.freq = freq if freq > 0
+      
+      return word
+    end
+    
+    # Do not clean and/or strip spaces, as the raw text is important for
+    #   Defn and ArticleScraper.
+    def self.scrape_ruby_tag(tag,missingno: nil,url: nil)
+      # First, try <rb> tags.
+      kanji = tag.css('rb')
+      # Second, try text nodes.
+      kanji = tag.search('./text()') if kanji.length < 1
+      # Third, try non-<rt> tags, in case of being surrounded by <span>, <b>, etc.
+      kanji = tag.search("./*[not(name()='rt')]") if kanji.length < 1
+      
+      raise ScrapeError,"no kanji at URL[#{url}] in tag[#{tag}]" if kanji.length < 1
+      raise ScrapeError,"too many kanji at URL[#{url}] in tag[#{tag}]" if kanji.length > 1
+      
+      kanji = kanji[0].text
+      kana = tag.css('rt')
+      
+      raise ScrapeError,"no kana at URL[#{url}] in tag[#{tag}]" if kana.length < 1
+      raise ScrapeError,"too many kana at URL[#{url}] in tag[#{tag}]" if kana.length > 1
+      
+      kana = kana[0].text
+      
+      if !missingno.nil?()
+        # Check kana first, since this is the typical scenario.
+        # - https://www3.nhk.or.jp/news/easy/k10012331311000/k10012331311000.html
+        # - '窓' in '（８）窓を開けて外の空気を入れましょう'
+        if Util.empty_web_str?(kana)
+          kana = missingno.kana_from_kanji(kanji)
+          
+          if !Util.empty_web_str?(kana)
+            Util.warn("using missingno for kana[#{kana}] from kanji[#{kanji}]")
+          end
+        elsif Util.empty_web_str?(kanji)
+          kanji = missingno.kanji_from_kana(kana)
+          
+          if !Util.empty_web_str?(kanji)
+            Util.warn("using missingno for kanji[#{kanji}] from kana[#{kana}]")
+          end
+        end
+      end
+      
+      raise ScrapeError,"empty kanji at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kanji)
+      raise ScrapeError,"empty kana at URL[#{url}] in tag[#{tag}]" if Util.empty_web_str?(kana)
+      
+      word = Word.new(kana: kana,kanji: kanji)
+      
+      return word
+    end
+    
+    # Do not clean and/or strip spaces, as the raw text is important for
+    #   Defn and ArticleScraper.
+    def self.scrape_text_node(tag,url: nil)
+      text = tag.text
+      
+      # No error; empty text is fine (not strictly kanji/kana only).
+      return nil if Util.empty_web_str?(text)
       
-      freq = hash[:freq].to_i() # nil is okay
-      word.freq = freq if freq >= 0
+      word = Word.new(unknown: text)
       
       return word
     end
     
     def kanji?()
-      return @word != @kana
+      return !Util.empty_web_str?(@kanji)
     end
     
-    def no_kanji?()
-      return !kanji?()
+    def word()
+      return kanji?() ? @kanji : @kana
     end
     
     def to_s()
       s = ''.dup()
       
-      s << @word
-      s << " | kana: #{@kana}"
-      s << " | freq: #{@freq}"
-      s << " | kanji? #{kanji?() ? 'yes' : 'no'}"
+      s << "'#{@key}': "
+      s << "{ kanji=>'#{@kanji}'"
+      s << ", kana=>'#{@kana}'"
+      s << ", freq=>#{@freq}"
+      s << ", defn=>'#{@defn.to_s().gsub("\n",'\\n')}'"
+      s << ", eng=>'#{@eng}'"
+      s << ' }'
       
       return s
     end

data/lib/nhkore.rb

@@ -21,60 +21,59 @@
 #++
 
 
-IS_TESTING = ($0 == __FILE__)
+TESTING = ($0 == __FILE__)
 
-if IS_TESTING
+if TESTING
   require 'rubygems'
   require 'bundler/setup'
 end
 
+require 'nhkore/app'
+require 'nhkore/article'
+require 'nhkore/article_scraper'
+require 'nhkore/cleaner'
+require 'nhkore/defn'
+require 'nhkore/dict'
+require 'nhkore/dict_scraper'
+require 'nhkore/entry'
+require 'nhkore/error'
+require 'nhkore/fileable'
+require 'nhkore/missingno'
+require 'nhkore/news'
+require 'nhkore/polisher'
+require 'nhkore/scraper'
+require 'nhkore/search_link'
+require 'nhkore/search_scraper'
+require 'nhkore/sifter'
+require 'nhkore/splitter'
+require 'nhkore/util'
+require 'nhkore/variator'
 require 'nhkore/version'
 require 'nhkore/word'
 
+require 'nhkore/cli/bing_cmd'
+require 'nhkore/cli/fx_cmd'
+require 'nhkore/cli/get_cmd'
+require 'nhkore/cli/news_cmd'
+require 'nhkore/cli/sift_cmd'
 
-require 'nokogiri'
-require 'psychgus'
-
-# 2 files:
-# nhk_news_web_easy.yml
-# (only Dec 2019 as an example; more in release)
-# (add comment about above in this file)
-# - 2019-12-25 13:10 JST: (append 1,2,... if duplicate)
-#   - url: <url>
-#   - md5: <md5sum of content only (in case of ads)>
-#   - words:
-#     - word (kanji/kana):
-#       - kana:
-#       - freq:
-# (only Dec 2019 as an example; more in release)
-# nhk_news_web_easy_core_<search criteria>.csv
-#   word, kana, freq
-#   (sorted by freq, word, or kana [desc/asc])
-
-# nhkore --date '2019-12-01...2019-12-11'
-# nhkore --date '2019-12'
-# nhkore --date '12'    (Dec of this year)
-# nhkore --date '12-01' (Dec 1 of this year)
-
-class Article
-  attr_accessor :datetime
-  attr_accessor :futsuu_url
-  attr_accessor :md5
-  attr_accessor :url
-  attr_accessor :words
-end
-
-word = NHKore::Word.new(kana: 'banana')
-puts word.to_yaml
 
 ###
 # @author Jonathan Bradley Whited (@esotericpig)
 # @since  0.1.0
 ###
 module NHKore
+  # @since 0.2.0
+  def self.run(args=ARGV)
+    app = App.new(args)
+    
+    begin
+      app.run()
+    rescue CLIError => e
+      puts "Error: #{e}"
+      exit 1
+    end
+  end
 end
 
-# https://www3.nhk.or.jp/news/easy
-# https://www.google.com/search?q=site:https://www3.nhk.or.jp/news/easy/&tbas=0&tbs=cdr:1,cd_min:1/1/2019,cd_max:12/31/2019,sbd:1&tbm=nws&sxsrf=ALeKk01oebeT0hWvNro-vDn7WGYkRe0kxw:1582396168988&ei=CHNRXt7dO5KR0QSTnZGYCw&start=20&sa=N&ved=0ahUKEwjega-M5eXnAhWSSJQKHZNOBLM4ChDy0wMIag&biw=1028&bih=672&dpr=1.3
-
-#NHKore::App.new().run() if IS_TESTING
+NHKore.run() if TESTING