nhkore 0.3.7 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b9464ae2a62f0c9cc797f2f70028d2b7afd6f0677a431c54e5453690175ca29
-  data.tar.gz: 577987179a9001926629f1efd8e8d39fb61ad62543d24ea1caa4f1fe063fd1a4
+  metadata.gz: c63efbc2f65cfe83c7b55e53a0dfca329c2aded4c22ae05c2fb50583876452b4
+  data.tar.gz: 87c5116e11cb7e2dd4a5cdb86d6fc1a80ea58dd4efa7bc27ad448c25c4fad724
 SHA512:
-  metadata.gz: '09d90011d4d641ea54c9dd7ebc8fd95efc8f7e68211e4322c1f3294e15a21303147de1eea2532694d5a01caaaf3c73f9a5172479193113be86b5b7a9fd08b910'
-  data.tar.gz: 65512547e6ee13503b345402e2eb1ba799e492131975f518cc96576a684bc97d48efb0f9c22787518f8dc62998e44fba1089ca2a1687b98e0533a489091e61c1
+  metadata.gz: 68eb93da6d8f5c8ba3c4c58e0a9a71803dd4eefc6063df4ead9f0d06c0f1ba59892f5ddb43a9735c30ceaf85db63ed80c1b155bac1d5f0daf73f9cebbc7f6c6e
+  data.tar.gz: 33e9f4f770bceb2c0eb5d6d62781af400bc2b66f4ba4d4092b01b224bc365edef98cc47032f4b2389e04664f6cabcd2c02024139971bee207a121570805a6015

data/CHANGELOG.md

@@ -1,8 +1,30 @@
 # Changelog | NHKore
 
-Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+All notable changes to this project will be documented in this file.
+
+Format is based on [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0),
+and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
+
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.8...HEAD)
+-
+
+
+## [v0.3.8] - 2021-06-26
+
+### Fixed
+- Fixed `App#refresh_cmd()` to also copy Cri's `default_proc` to the new Hash for the command options.
+- Fixed to check for non-strings for JSON & URI.
+    - For JSON, convert `StringIO` to string in `DictScraper.scrape()`.
+    - For URL, convert URL using `URI()` because `URI.parse()` will crash with a non-string (URI object) in `Scraper.open_url()`.
+- Fixed to scrape multiple HTML Ruby tag words (instead of just 1).
+    - I thought multiple Ruby bases/texts (`<rb>`/`<rt>`) were invalid, but after running into the article below and checking the HTML with a validator, it's actually valid HTML:
+          - https://www3.nhk.or.jp/news/easy/k10012759201000/k10012759201000.html
+          - No previous articles/URLs ran into this problem (would have raised an error), so it should only be a problem with this specific, new article.
+
+### Changed
+- Formatted/Linted all code using RuboCop.
+- Updated Gems.
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.7...HEAD)
 
 ## [v0.3.7] - 2020-11-07
 
@@ -20,6 +42,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
         - https://www3.nhk.or.jp/news/easy/k10012639271000/k10012639271000.html
             - `第３のビール` should have HTML ruby tags around *第*
 
+
 ## [v0.3.6] - 2020-08-18
 
 ### Added
@@ -35,6 +58,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Ignored `/cgi2.*enqform/` URLs from SearchScraper (Bing)
 - Added more detail to dictionary error in ArticleScraper
 
+
 ## [v0.3.5] - 2020-05-04
 
 ### Added
@@ -45,6 +69,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fixed URLs stored in YAML data to always be of type String (not URI)
     - This initially caused a problem in DictScraper.parse_url() from ArticleScraper, but fixed it for all data
 
+
 ## [v0.3.4] - 2020-04-25
 
 ### Added
@@ -66,12 +91,14 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
         - `-d '3-3'`
         - `-d '3'`
 
+
 ## [v0.3.3] - 2020-04-23
 
 ### Added
 - Added JSON support to Sifter & SiftCmd.
 - Added use of `attr_bool` Gem for `attr_accessor?` & `attr_reader?`.
 
+
 ## [v0.3.2] - 2020-04-22
 
 ### Added
@@ -95,6 +122,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - ArticleScraper
     - Renamed `mode` param to `strict`. `mode` was overshadowing File.open()'s in Scraper.
 
+
 ## [v0.3.1] - 2020-04-20
 
 ### Changed
@@ -112,6 +140,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - BingScraper
     - Fixed possible RSS infinite loop.
 
+
 ## [v0.3.0] - 2020-04-12
 
 ### Added
@@ -146,7 +175,9 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
         - ignore empty filenames in the Zip for safety.
         - ask to overwrite files instead of erroring.
 
+
 ## [v0.2.0] - 2020-04-01
+
 First working version.
 
 ### Added
@@ -182,7 +213,9 @@ First working version.
 - test/nhkore_tester.rb
     - Renamed to `test/nhkore/test_helper.rb`
 
+
 ## [v0.1.0] - 2020-02-24
+
 ### Added
 - .gitignore
 - CHANGELOG.md

data/Gemfile

@@ -1,24 +1,6 @@
 # encoding: UTF-8
 # frozen_string_literal: true
 
-#--
-# This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
-#++
-
 
 source 'https://rubygems.org'
 

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    nhkore (0.3.7)
+    nhkore (0.3.8)
       attr_bool (~> 0.2)
       bimyou_segmenter (~> 1.2)
       cri (~> 2.15)
-      down (~> 5.1)
+      down (~> 5.2)
       highline (~> 2.0)
       http-cookie (~> 1.0)
       japanese_deinflector (~> 0.0)
-      nokogiri (~> 1.10)
+      nokogiri (~> 1.11)
       psychgus (~> 1.3)
       public_suffix (~> 4.0)
       rainbow (~> 3.0)
       rubyzip (~> 2.3)
       tiny_segmenter (~> 0.0)
-      tty-progressbar (~> 0.17)
+      tty-progressbar (~> 0.18)
       tty-spinner (~> 0.9)
 
 GEM
@@ -23,64 +23,67 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    attr_bool (0.2.1)
+    attr_bool (0.2.2)
     bimyou_segmenter (1.2.0)
-    cri (2.15.10)
+    cri (2.15.11)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    down (5.2.0)
+    down (5.2.2)
       addressable (~> 2.5)
     highline (2.0.3)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     japanese_deinflector (0.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    psych (3.2.0)
-    psychgus (1.3.3)
+    mini_portile2 (2.5.3)
+    minitest (5.14.4)
+    nokogiri (1.11.7)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    psych (4.0.1)
+    psychgus (1.3.4)
       psych (>= 3.0)
     public_suffix (4.0.6)
+    racc (1.5.2)
     rainbow (3.0.0)
-    rake (13.0.1)
-    raketeer (0.2.9)
+    rake (13.0.3)
+    raketeer (0.2.13)
       rake
-    rdoc (6.2.1)
-    redcarpet (3.5.0)
+    rdoc (6.3.1)
+    redcarpet (3.5.1)
     rubyzip (2.3.0)
-    strings-ansi (0.1.0)
+    strings-ansi (0.2.0)
     tiny_segmenter (0.0.6)
     tty-cursor (0.7.1)
-    tty-progressbar (0.17.0)
-      strings-ansi (~> 0.1.0)
+    tty-progressbar (0.18.2)
+      strings-ansi (~> 0.2)
       tty-cursor (~> 0.7)
-      tty-screen (~> 0.7)
-      unicode-display_width (~> 1.6)
+      tty-screen (~> 0.8)
+      unicode-display_width (>= 1.6, < 3.0)
     tty-screen (0.8.1)
     tty-spinner (0.9.3)
       tty-cursor (~> 0.7)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unicode-display_width (1.7.0)
-    yard (0.9.25)
-    yard_ghurt (1.2.0)
+    unicode-display_width (2.0.0)
+    yard (0.9.26)
+    yard_ghurt (1.2.1)
       rake
+      yard
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 2.1)
+  bundler (~> 2.2)
   minitest (~> 5.14)
   nhkore!
   rake (~> 13.0)
   raketeer (~> 0.2)
-  rdoc (~> 6.2)
+  rdoc (~> 6.3)
   redcarpet (~> 3.5)
   yard (~> 0.9)
   yard_ghurt (~> 1.2)
 
 BUNDLED WITH
-   2.1.4
+   2.2.20

data/README.md

@@ -435,18 +435,18 @@ require 'nhkore/scraper'
 s = NHKore::Scraper.new('https://www3.nhk.or.jp/news/easy/',
   open_timeout: 300, # Open timeout in seconds (default: nil)
   read_timeout: 300, # Read timeout in seconds (default: nil)
-  
+
   # Maximum number of times to retry the URL
   # - default: 3
   # - Open/connect will fail a couple of times on a bad/slow internet connection.
   max_retries: 10,
-  
+
   # Maximum number of redirects allowed.
   # - default: 3
   # - You can set this to nil or -1, but I recommend using a number
   #   for safety (infinite-loop attack).
   max_redirects: 1,
-  
+
   # How to check redirect URLs for safety.
   # - default: :strict
   # - nil      => do not check
@@ -455,7 +455,7 @@ s = NHKore::Scraper.new('https://www3.nhk.or.jp/news/easy/',
   # - :strict  => check the scheme and domain
   #               (i.e., if https://bing.com, redirect URL must be https://bing.com)
   redirect_rule: :lenient,
-  
+
   # Set the HTTP header field 'cookie' from the 'set-cookie' response.
   # - default: false
   # - Currently uses the 'http-cookie' Gem.
@@ -463,7 +463,7 @@ s = NHKore::Scraper.new('https://www3.nhk.or.jp/news/easy/',
   # - Necessary for Search Engines or other sites that require cookies
   #   in order to block bots.
   eat_cookie: true,
-  
+
   # Set HTTP header fields.
   # - default: nil
   # - Necessary for Search Engines or other sites that try to block bots.
@@ -526,9 +526,9 @@ doc = ss.html_doc()
 
 doc.css('a').each() do |anchor|
   link = anchor['href']
-  
+
   next if ss.ignore_link?(link,cleaned: false)
-  
+
   if link.include?('https://www3.nhk')
     puts link
   end
@@ -549,9 +549,9 @@ page_num  = 1
 
 while !next_page.empty?()
   puts "Page #{page_num += 1}: #{next_page.count}"
-  
+
   bs = NHKore::BingScraper.new(:yasashii,url: next_page.url)
-  
+
   next_page = bs.scrape(slinks,next_page)
 end
 
@@ -570,24 +570,24 @@ require 'time'
 
 as = NHKore::ArticleScraper.new(
   'https://www3.nhk.or.jp/news/easy/k10011862381000/k10011862381000.html',
-  
+
   # If false, scrape the article leniently (for older articles which
   # may not have certain tags, etc.).
   # - default: true
   strict: false,
-  
+
   # {Dict} to use as the dictionary for words (Easy articles).
   # - default: :scrape
   # - nil     => don't scrape/use it (necessary for Regular articles)
   # - :scrape => auto-scrape it using {DictScraper}
   # - {Dict}  => your own {Dict}
   dict: nil,
-  
+
   # Date time to use as a fallback if the article doesn't have one
   # (for older articles).
   # - default: nil
   datetime: Time.new(2020,2,2),
-  
+
   # Year to use as a fallback if the article doesn't have one
   # (for older articles).
   # - default: nil
@@ -624,7 +624,7 @@ require 'nhkore/dict_scraper'
 url = 'https://www3.nhk.or.jp/news/easy/k10011862381000/k10011862381000.html'
 ds  = NHKore::DictScraper.new(
   url,
-  
+
   # Change the URL appropriately to the dictionary URL.
   # - default: true
   parse_url: true,
@@ -637,13 +637,13 @@ dict = ds.scrape()
 
 dict.entries.each() do |key,entry|
   entry.id
-  
+
   entry.defns.each() do |defn|
     defn.hyoukis.each() {|hyouki| }
     defn.text
     defn.words.each() {|word| }
   end
-  
+
   puts entry.build_hyouki()
   puts entry.build_defn()
   puts '---'
@@ -789,13 +789,13 @@ JPN = ['桜','ぶ','ブ']
 
 def fmt_jpn()
   fmt = []
-  
+
   JPN.each() do |x|
     x = yield(x)
     x = x ? "\u2B55" : Util::JPN_SPACE unless x.is_a?(String)
     fmt << x
   end
-  
+
   return "[ #{fmt.join(' | ')} ]"
 end
 
@@ -879,7 +879,7 @@ Releasing new HTML file for website:
 [GNU LGPL v3+](LICENSE.txt)
 
 > NHKore (<https://github.com/esotericpig/nhkore>)  
-> Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)  
+> Copyright (c) 2020-2021 Jonathan Bradley Whited  
 > 
 > NHKore is free software: you can redistribute it and/or modify  
 > it under the terms of the GNU Lesser General Public License as published by  

data/Rakefile

@@ -1,24 +1,6 @@
 # encoding: UTF-8
 # frozen_string_literal: true
 
-#--
-# This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
-#++
-
 
 require 'bundler/gem_tasks'
 
@@ -42,20 +24,20 @@ CLOBBER.include('doc/',File.join(PKG_DIR,''))
 task default: [:test]
 
 desc 'Generate documentation (YARDoc)'
-task :doc => [:yard,:yard_gfm_fix] do |task|
+task doc: %i[yard yard_gfm_fix] do |task|
 end
 
 desc "Package '#{File.join(NHKore::Util::CORE_DIR,'')}' data as a Zip file into '#{File.join(PKG_DIR,'')}'"
 task :pkg_core do |task|
   mkdir_p PKG_DIR
-  
+
   pattern = File.join(NHKore::Util::CORE_DIR,'*.{csv,html,json,yml}')
   zip_file = File.join(PKG_DIR,'nhkore-core.zip')
-  
-  sh 'zip','-9rv',zip_file,*Dir.glob(pattern).sort()
+
+  sh 'zip','-9rv',zip_file,*Dir.glob(pattern).sort
 end
 
-Rake::TestTask.new() do |task|
+Rake::TestTask.new do |task|
   task.libs = ['lib','test']
   task.pattern = File.join('test','**','*_test.rb')
   task.description += ": '#{task.pattern}'"
@@ -68,73 +50,77 @@ end
 desc "Update '#{File.join(NHKore::Util::CORE_DIR,'')}' files for release"
 task :update_core do |task|
   require 'highline'
-  
-  CONTINUE_MSG = "\nContinue (y/n)? "
-  
+
+  continue_msg = "\nContinue (y/n)? "
+
   cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
-  hl = HighLine.new()
-  
+  hl = HighLine.new
+
   next unless sh(*cmd,'se','ez','bing')
-  next unless hl.agree(CONTINUE_MSG)
+  next unless hl.agree(continue_msg)
   puts
-  
-  next unless sh(*cmd,'news','-s','100','ez')
-  next unless hl.agree(CONTINUE_MSG)
+
+  next unless sh(*cmd,'news','-s','500','ez')
+  next unless hl.agree(continue_msg)
   puts
-  
+
   next unless sh(*cmd,'sift','-e','csv' ,'ez')
+  puts
   next unless sh(*cmd,'sift','-e','html','ez')
+  puts
   next unless sh(*cmd,'sift','-e','json','ez')
+  puts
   next unless sh(*cmd,'sift','-e','yml' ,'ez')
+  puts
 end
 
 # @since 0.3.6
 desc 'Update showcase file for release'
 task :update_showcase do |task|
   require 'highline'
-  
-  SHOWCASE_FILE = File.join('.','nhkore-ez.html')
-  
-  hl = HighLine.new()
-  
+
+  showcase_file = File.join('.','nhkore-ez.html')
+
+  hl = HighLine.new
+
   next unless sh('ruby','-w','./lib/nhkore.rb',
     'sift','ez','--no-eng',
-    '--out',SHOWCASE_FILE,
+    '--out',showcase_file,
   )
-  
+
   next unless hl.agree("\nMove the file (y/n)? ")
   puts
-  next unless sh('mv','-iv',SHOWCASE_FILE,
+  next unless sh('mv','-iv',showcase_file,
     File.join('..','esotericpig.github.io','showcase',''),
   )
 end
 
-YARD::Rake::YardocTask.new() do |task|
+YARD::Rake::YardocTask.new do |task|
   task.options += ['--template-path',File.join('yard','templates')]
   task.options += ['--title',"NHKore v#{NHKore::VERSION} Doc"]
 end
 
 # Execute "rake yard_gfm_fix" for production.
 # Execute "rake yard_gfm_fix[true]" for testing locally.
-YardGhurt::GFMFixTask.new() do |task|
+YardGhurt::GFMFixTask.new do |task|
   task.arg_names = [:dev]
   task.dry_run = false
   task.fix_code_langs = true
   task.md_files = ['index.html']
-  
-  task.before = Proc.new() do |task,args|
+
+  task.before = proc do |t,args|
     # Delete this file as it's never used (index.html is an exact copy).
-    YardGhurt::Util.rm_exist(File.join(task.doc_dir,'file.README.html'))
-    
+    YardGhurt::Util.rm_exist(File.join(t.doc_dir,'file.README.html'))
+
     # Root dir of my GitHub Page for CSS/JS.
-    GHP_ROOT = YardGhurt::Util.to_bool(args.dev) ? '../../esotericpig.github.io' : '../../..'
-    
-    task.css_styles << %Q(<link rel="stylesheet" type="text/css" href="#{GHP_ROOT}/css/prism.css" />)
-    task.js_scripts << %Q(<script src="#{GHP_ROOT}/js/prism.js"></script>)
+    ghp_root = YardGhurt::Util.to_bool(args.dev) ? '../../esotericpig.github.io' : '../../..'
+
+    t.css_styles << %Q(<link rel="stylesheet" type="text/css" href="#{ghp_root}/css/prism.css" />)
+    t.js_scripts << %Q(<script src="#{ghp_root}/js/prism.js"></script>)
   end
 end
 
 # Probably not useful for others.
-YardGhurt::GHPSyncTask.new() do |task|
+YardGhurt::GHPSyncTask.new do |task|
   task.ghp_dir = '../esotericpig.github.io/docs/nhkore/yardoc'
 end