nhkore 0.1.0 → 0.2.0

data/lib/nhkore/cleaner.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/util'
+require 'nhkore/word'
+
+
+module NHKore
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class Cleaner
+    def begin_clean(str)
+      return str
+    end
+    
+    def clean(str)
+      str = begin_clean(str)
+      str = end_clean(str)
+      
+      return str
+    end
+    
+    def self.clean_any(obj,cleaners)
+      return nil if obj.nil?()
+      
+      cleaners = Array(cleaners)
+      
+      return obj if cleaners.empty?()
+      
+      if obj.is_a?(Word)
+        obj = Word.new(
+          kana: clean_any(obj.kana,cleaners),
+          kanji: clean_any(obj.kanji,cleaners),
+          word: obj
+        )
+      else # String
+        cleaners.each() do |cleaner|
+          obj = cleaner.clean(obj)
+        end
+      end
+      
+      return obj
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BasicCleaner < Cleaner
+    def end_clean(str)
+      # This is very simple, as Splitter will split on punctuation,
+      #   and Polisher will remove the leftover punctuation, digits, etc.
+      # If this is stricter, then errors will be raised in ArticleScraper's
+      #   scrape_dicwin_word() & scrape_ruby_word().
+      
+      str = Util.unspace_web_str(str) # Who needs space in Japanese?
+      
+      return str
+    end
+  end
+  
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  class BestCleaner < BasicCleaner
+  end
+end

data/lib/nhkore/cli/bing_cmd.rb

@@ -0,0 +1,220 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'nhkore/error'
+require 'nhkore/search_link'
+require 'nhkore/search_scraper'
+require 'nhkore/util'
+
+
+module NHKore
+module CLI
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  module BingCmd
+    def build_bing_cmd()
+      app = self
+      
+      @bing_cmd = @app_cmd.define_command() do
+        name    'bing'
+        usage   'bing [OPTIONS] [COMMAND]...'
+        aliases :b
+        summary "Search bing.com for links to NHK News Web (Easy) (aliases: #{app.color_alias('b')})"
+        
+        description <<-EOD
+          Search bing.com for links to NHK News Web (Easy) &
+          save to folder: #{SearchLinks::DEFAULT_DIR}
+        EOD
+        
+        option :i,:in,<<-EOD,argument: :required,transform: -> (value) do
+          HTML file to read instead of URL (for offline testing and/or slow internet;
+          see '--show-urls' option)
+        EOD
+          app.check_empty_opt(:in,value)
+        end
+        option :o,:out,<<-EOD,argument: :required,transform: -> (value) do
+          'directory/file' to save links to; if you only specify a directory or a file, it will attach the
+          appropriate default directory/file name
+          (defaults: #{SearchLinks::DEFAULT_BING_YASASHII_FILE}, #{SearchLinks::DEFAULT_BING_FUTSUU_FILE})
+        EOD
+          app.check_empty_opt(:out,value)
+        end
+        option :r,:results,'number of results per page to request from Bing',argument: :required,
+          default: SearchScraper::DEFAULT_RESULT_COUNT,transform: -> (value) do
+          value = value.to_i()
+          value = 1 if value < 1
+          value
+        end
+        option nil,:'show-count',<<-EOD
+          show the number of links scraped and exit;
+          useful for manually writing/updating scripts (but not for use in a variable);
+          implies '--dry-run' option
+        EOD
+        option nil,:'show-urls',<<-EOD do |value,cmd|
+          show the URLs used when scraping and exit; you can download these for offline testing and/or
+          slow internet (see '--in' option)
+        EOD
+          puts "Easy:    #{BingScraper.build_url(SearchScraper::YASASHII_SITE)}"
+          puts "Regular: #{BingScraper.build_url(SearchScraper::FUTSUU_SITE)}"
+          exit
+        end
+        
+        run do |opts,args,cmd|
+          puts cmd.help
+        end
+      end
+      
+      @bing_easy_cmd = @bing_cmd.define_command() do
+        name    'easy'
+        usage   'easy [OPTIONS] [COMMAND]...'
+        aliases :e,:ez
+        summary "Search for NHK News Web Easy (Yasashii) links (aliases: #{app.color_alias('e ez')})"
+        
+        description <<-EOD
+          Search for NHK News Web Easy (Yasashii) links &
+          save to file: #{SearchLinks::DEFAULT_BING_YASASHII_FILE}
+        EOD
+        
+        run do |opts,args,cmd|
+          app.refresh_cmd(opts,args,cmd)
+          app.run_bing_cmd(:yasashii)
+        end
+      end
+      
+      @bing_regular_cmd = @bing_cmd.define_command() do
+        name    'regular'
+        usage   'regular [OPTIONS] [COMMAND]...'
+        aliases :r,:reg
+        summary "Search for NHK News Web Regular (Futsuu) links (aliases: #{app.color_alias('r reg')})"
+        
+        description <<-EOD
+          Search for NHK News Web Regular (Futsuu) links &
+          save to file: #{SearchLinks::DEFAULT_BING_FUTSUU_FILE}
+        EOD
+        
+        run do |opts,args,cmd|
+          app.refresh_cmd(opts,args,cmd)
+          app.run_bing_cmd(:futsuu)
+        end
+      end
+    end
+    
+    def run_bing_cmd(type)
+      @cmd_opts[:dry_run] = true if @cmd_opts[:show_count]
+      
+      build_in_file(:in)
+      
+      case type
+      when :futsuu
+        build_out_file(:out,default_dir: SearchLinks::DEFAULT_DIR,default_filename: SearchLinks::DEFAULT_BING_FUTSUU_FILENAME)
+      when :yasashii
+        build_out_file(:out,default_dir: SearchLinks::DEFAULT_DIR,default_filename: SearchLinks::DEFAULT_BING_YASASHII_FILENAME)
+      else
+        raise ArgumentError,"invalid type[#{type}]"
+      end
+      
+      return unless check_in_file(:in,empty_ok: true)
+      return unless check_out_file(:out)
+      
+      dry_run = @cmd_opts[:dry_run]
+      in_file = @cmd_opts[:in]
+      out_file = @cmd_opts[:out]
+      result_count = @cmd_opts[:results]
+      result_count = SearchScraper::DEFAULT_RESULT_COUNT if result_count.nil?()
+      show_count = @cmd_opts[:show_count]
+      
+      start_spin('Scraping bing.com') unless show_count
+      
+      is_file = !in_file.nil?()
+      links = nil
+      new_links = [] # For --dry-run
+      next_page = NextPage.new()
+      page_count = 0
+      page_num = 1
+      url = in_file # nil will use default URL, else a file
+      
+      # Load previous links for 'scraped?' vars.
+      if File.exist?(out_file)
+        links = SearchLinks.load_file(out_file)
+      else
+        links = SearchLinks.new()
+      end
+      
+      links_count = links.length
+      
+      if show_count
+        scraped_count = 0
+        
+        links.links.values.each() do |link|
+          scraped_count += 1 if link.scraped?()
+        end
+        
+        puts "#{scraped_count} of #{links_count} links scraped."
+        
+        return
+      end
+      
+      # Do a range to prevent an infinite loop. Ichiman!
+      (0..10000).each() do
+        scraper = BingScraper.new(type,count: result_count,is_file: is_file,url: url,**@scraper_kargs)
+        
+        next_page = scraper.scrape(links,next_page)
+        
+        new_links.concat(links.links.values[links_count..-1])
+        links_count = links.length
+        page_count = next_page.count if next_page.count > 0
+        
+        update_spin_detail(" (page=#{page_num}, count=#{page_count}, links=#{links.length}, " +
+          "new_links=#{new_links.length})")
+        
+        break if next_page.empty?()
+        
+        page_num += 1
+        url = next_page.url
+        
+        sleep_scraper()
+      end
+      
+      stop_spin()
+      puts
+      
+      puts 'Last URL scraped:'
+      puts "> #{url}"
+      puts
+      
+      if dry_run
+        new_links.each() do |link|
+          puts link.to_s(mini: true)
+        end
+      else
+        links.save_file(out_file)
+        
+        puts 'Saved scraped links to file:'
+        puts "> #{out_file}"
+      end
+    end
+  end
+end
+end

data/lib/nhkore/cli/fx_cmd.rb

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+module NHKore
+module CLI
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  module FXCmd
+    def build_fx_cmd()
+      app = self
+      
+      @fx_cmd = @app_cmd.define_command() do
+        name    'fx'
+        usage   'fx [OPTIONS] [COMMAND]...'
+        summary 'Test spinner/progress special effects (for running long tasks)'
+        
+        description <<-EOD
+          Test if the special effects work on your command line:\n
+          - #{App::NAME} [-c/-X] fx
+        EOD
+        
+        flag :a,:all,'test all special effects regardless of global options'
+        
+        run do |opts,args,cmd|
+          app.refresh_cmd(opts,args,cmd)
+          app.run_fx_cmd()
+        end
+      end
+    end
+    
+    def run_fx_cmd()
+      test_fx_progress_bar()
+      test_fx_spinner()
+    end
+    
+    def test_fx_progress_bar()
+      bars = nil
+      
+      if @cmd_opts[:all]
+        bars = [:default,:classic,:no]
+      else
+        bars = [@progress_bar]
+      end
+      
+      bars.each() do |bar|
+        name = (bars.length == 1) ? 'User' : bar.to_s().capitalize()
+        bar = build_progress_bar("Testing #{name} progress",download: false,type: bar)
+        
+        bar.start()
+        
+        0.upto(99) do
+          sleep(0.05)
+          bar.advance()
+        end
+        
+        bar.finish()
+      end
+    end
+    
+    def test_fx_spinner()
+      app_spinner = @spinner
+      spinners = nil
+      
+      if @cmd_opts[:all]
+        spinners = {
+          default: App::DEFAULT_SPINNER,
+          classic: App::CLASSIC_SPINNER,
+          no: App::NO_SPINNER
+        }
+      else
+        spinners = {
+          user: app_spinner
+        }
+      end
+      
+      spinners.each() do |name,spinner|
+        @spinner = spinner
+        
+        start_spin("Testing #{name.to_s().capitalize()} spinner")
+        
+        1.upto(3) do |i|
+          sleep(1.1)
+          update_spin_detail(" (#{i}/3)")
+        end
+        
+        stop_spin()
+      end
+      
+      # Reset back to users'.
+      @spinner = app_spinner
+    end
+  end
+end
+end

data/lib/nhkore/cli/get_cmd.rb

@@ -0,0 +1,153 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+require 'down/net_http'
+require 'tempfile'
+require 'zip'
+
+require 'nhkore/util'
+
+
+module NHKore
+module CLI
+  ###
+  # @author Jonathan Bradley Whited (@esotericpig)
+  # @since  0.2.0
+  ###
+  module GetCmd
+    DEFAULT_GET_CHUNK_SIZE = 4 * 1024
+    DEFAULT_GET_URL_LENGTH = 5_000_000 # Just a generous estimation used as a fallback; may be outdated
+    GET_URL_FILENAME = 'nhkore-core.zip'
+    GET_URL = "https://github.com/esotericpig/nhkore/releases/latest/download/#{GET_URL_FILENAME}"
+    
+    def build_get_cmd()
+      app = self
+      
+      @get_cmd = @app_cmd.define_command() do
+        name    'get'
+        usage   'get [OPTIONS] [COMMAND]...'
+        aliases :g
+        summary "Download NHKore's pre-scraped files from the latest release (aliases: #{app.color_alias('g')})"
+        
+        description <<-EOD
+          Download NHKore's pre-scraped files from the latest release &
+          save to folder: #{Util::CORE_DIR}
+          
+          Note: the latest NHK articles may not have been scraped yet.
+        EOD
+        
+        option :o,:out,'directory to save downloaded files to',argument: :required,default: Util::CORE_DIR,
+          transform: -> (value) do
+          app.check_empty_opt(:out,value)
+        end
+        flag nil,:'show-url','show download URL and exit (for downloading manually)' do |value,cmd|
+          puts GET_URL
+          exit
+        end
+        
+        run do |opts,args,cmd|
+          app.refresh_cmd(opts,args,cmd)
+          app.run_get_cmd()
+        end
+      end
+    end
+    
+    def run_get_cmd()
+      build_out_dir(:out,default_dir: Util::CORE_DIR)
+      
+      return unless check_out_dir(:out)
+      
+      chunk_size = DEFAULT_GET_CHUNK_SIZE
+      down = nil
+      dry_run = @cmd_opts[:dry_run]
+      force = @cmd_opts[:force]
+      max_retries = @scraper_kargs[:max_retries]
+      max_retries = 3 if max_retries.nil?()
+      out_dir = @cmd_opts[:out]
+      
+      begin
+        start_spin('Opening URL')
+        
+        begin
+          down = Down::NetHttp.open(GET_URL,rewindable: false,**@scraper_kargs)
+        rescue Down::ConnectionError
+          raise if (max_retries -= 1) < 0
+          retry
+        end
+        
+        stop_spin()
+        
+        return if dry_run
+        
+        Tempfile.create([App::NAME,'.zip'],binmode: true) do |file|
+          puts
+          puts 'Downloading to temp file:'
+          puts "> #{file.path}"
+          puts
+          
+          len = down.size
+          len = DEFAULT_GET_LENGTH if len.nil?()
+          bar = build_progress_bar("Downloading #{GET_URL_FILENAME}",download: true,total: len)
+          
+          bar.start()
+          
+          while !down.eof?()
+            file.write(down.read(chunk_size))
+            bar.advance(chunk_size)
+          end
+          
+          down.close()
+          file.close()
+          bar.finish()
+          
+          start_spin("Extracting #{GET_URL_FILENAME}")
+          
+          Zip.on_exists_proc = force # true will force overwriting files on extract()
+          
+          Zip::File.open(file) do |zip_file|
+            zip_file.each() do |entry|
+              if !entry.name_safe?()
+                raise ZipError,"unsafe entry name[#{entry.name}] in Zip file"
+              end
+              
+              name = File.basename(entry.name)
+              
+              update_spin_detail(" (file=#{name})")
+              
+              entry.extract(File.join(out_dir,name))
+            end
+          end
+          
+          stop_spin()
+          puts
+          
+          puts "Extracted #{GET_URL_FILENAME} to directory:"
+          puts "> #{out_dir}"
+        end
+      ensure
+        down.close() if !down.nil?() && !down.closed?()
+      end
+    end
+  end
+end
+end