nhkore 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 445adf6e8abd4da9fd6dd25e9632d5f477b467f6ce8c3dcecae87e3f61305d98
-  data.tar.gz: ca812639ff1edd8da835f5bbb2cde403c9cb63e17568fb3ec367eec00605ec17
+  metadata.gz: 2b9464ae2a62f0c9cc797f2f70028d2b7afd6f0677a431c54e5453690175ca29
+  data.tar.gz: 577987179a9001926629f1efd8e8d39fb61ad62543d24ea1caa4f1fe063fd1a4
 SHA512:
-  metadata.gz: 392607205c53aa2a5dfcde244e5fa6137483d216dc27becf06c76798209d2dcf328f17abee2026d795207d4e783a23fd108e615525445f52ca6442560600cd42
-  data.tar.gz: 7a1219623b6645bbc633ba9c94e767dcf86be8852a7228c1d5ddd3936f61b884897f680369d4c9d9db5aba8ab4561048d59aed15cecf7ba05695c1957f31b0ea
+  metadata.gz: '09d90011d4d641ea54c9dd7ebc8fd95efc8f7e68211e4322c1f3294e15a21303147de1eea2532694d5a01caaaf3c73f9a5172479193113be86b5b7a9fd08b910'
+  data.tar.gz: 65512547e6ee13503b345402e2eb1ba799e492131975f518cc96576a684bc97d48efb0f9c22787518f8dc62998e44fba1089ca2a1687b98e0533a489091e61c1

data/.yardopts

@@ -0,0 +1,3 @@
+--files 'CHANGELOG.md,LICENSE.txt'
+--protected
+--readme 'README.md'

data/CHANGELOG.md

@@ -2,7 +2,23 @@
 
 Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.6...master)
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.7...HEAD)
+
+## [v0.3.7] - 2020-11-07
+
+### Changed
+- Updated Gem `attr_bool` to v0.2
+- Changed upper-case *'-V'* flag for *version* to be a lower-case *'-v'*
+    - Seems like a lot of apps/people expect this
+- Refactored/Formatted some code
+    - *nhkore.gemspec* especially
+- Added *samples/*, *Gemfile.lock*, and *.yardopts* to the files in *nhkore.gemspec*
+
+### Fixed
+- ArticleScraper
+    - Fixed to accept text nodes that have Kanji, due to bad article:
+        - https://www3.nhk.or.jp/news/easy/k10012639271000/k10012639271000.html
+            - `第３のビール` should have HTML ruby tags around *第*
 
 ## [v0.3.6] - 2020-08-18
 

data/Gemfile.lock

@@ -0,0 +1,86 @@
+PATH
+  remote: .
+  specs:
+    nhkore (0.3.7)
+      attr_bool (~> 0.2)
+      bimyou_segmenter (~> 1.2)
+      cri (~> 2.15)
+      down (~> 5.1)
+      highline (~> 2.0)
+      http-cookie (~> 1.0)
+      japanese_deinflector (~> 0.0)
+      nokogiri (~> 1.10)
+      psychgus (~> 1.3)
+      public_suffix (~> 4.0)
+      rainbow (~> 3.0)
+      rubyzip (~> 2.3)
+      tiny_segmenter (~> 0.0)
+      tty-progressbar (~> 0.17)
+      tty-spinner (~> 0.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    attr_bool (0.2.1)
+    bimyou_segmenter (1.2.0)
+    cri (2.15.10)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    down (5.2.0)
+      addressable (~> 2.5)
+    highline (2.0.3)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    japanese_deinflector (0.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.14.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    psych (3.2.0)
+    psychgus (1.3.3)
+      psych (>= 3.0)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    raketeer (0.2.9)
+      rake
+    rdoc (6.2.1)
+    redcarpet (3.5.0)
+    rubyzip (2.3.0)
+    strings-ansi (0.1.0)
+    tiny_segmenter (0.0.6)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.17.0)
+      strings-ansi (~> 0.1.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.7)
+      unicode-display_width (~> 1.6)
+    tty-screen (0.8.1)
+    tty-spinner (0.9.3)
+      tty-cursor (~> 0.7)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
+    yard (0.9.25)
+    yard_ghurt (1.2.0)
+      rake
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.1)
+  minitest (~> 5.14)
+  nhkore!
+  rake (~> 13.0)
+  raketeer (~> 0.2)
+  rdoc (~> 6.2)
+  redcarpet (~> 3.5)
+  yard (~> 0.9)
+  yard_ghurt (~> 1.2)
+
+BUNDLED WITH
+   2.1.4

data/lib/nhkore/app.rb

@@ -246,8 +246,7 @@ module NHKore
           app.scraper_kargs[:header] ||= {}
           app.scraper_kargs[:header]['user-agent'] = value
         end
-        # Big V, not small.
-        flag :V,:version,'show the version and exit' do |value,cmd|
+        flag :v,:version,'show the version and exit' do |value,cmd|
           app.show_version()
           exit
         end

data/lib/nhkore/article_scraper.rb

@@ -43,6 +43,8 @@ module NHKore
   # @since  0.2.0
   ###
   class ArticleScraper < Scraper
+    extend AttrBool::Ext
+    
     attr_reader :cleaners
     attr_accessor :datetime
     attr_accessor :dict
@@ -463,7 +465,10 @@ module NHKore
         return nil
       end
       
-      text = word.kana # Should be kana only
+      # Kanji only for:
+      # - https://www3.nhk.or.jp/news/easy/k10012639271000/k10012639271000.html
+      #   - '第３のビール'
+      text = word.word # Should usually be kana only
       
       result.add_text(text) # No cleaning; raw text
       
@@ -472,8 +477,9 @@ module NHKore
       return nil if text.empty?() # No error; empty text is fine here
       
       word = Word.new(
-        kana: text,
-        word: word
+        kana: clean(word.kana),
+        kanji: clean(word.kanji),
+        word: word,
       )
       
       return word

data/lib/nhkore/datetime_parser.rb

@@ -34,6 +34,8 @@ module NHKore
   # @since  0.3.4
   ###
   class DatetimeParser
+    extend AttrBool::Ext
+    
     # Order matters!
     FMTS = [
       '%Y-%m-%d %H:%M',

data/lib/nhkore/scraper.rb

@@ -35,6 +35,8 @@ module NHKore
   # @since  0.2.0
   ###
   class Scraper
+    extend AttrBool::Ext
+    
     DEFAULT_HEADER = {
       'user-agent' => UserAgents.sample(),
       'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;image/webp,image/apng,*/*;application/signed-exchange',

data/lib/nhkore/search_link.rb

@@ -34,6 +34,8 @@ module NHKore
   # @since  0.2.0
   ###
   class SearchLink
+    extend AttrBool::Ext
+    
     attr_reader :datetime
     attr_reader :futsuurl
     attr_accessor? :scraped

data/lib/nhkore/search_scraper.rb

@@ -45,6 +45,16 @@ module NHKore
     # - https://www3.nhk.or.jp/news/easy/article/disaster_heat.html
     YASASHII_REGEX = /\A[^\.]+\.#{Regexp.quote(YASASHII_SITE)}.+\.html?/i
     
+    IGNORE_LINK_REGEX = %r{
+      /about\.html?            # https://www3.nhk.or.jp/news/easy/about.html
+      |/movieplayer\.html?      # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
+      |/audio\.html?            # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
+      |/news/easy/index\.html?  # http://www3.nhk.or.jp/news/easy/index.html
+      # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
+      # https://www3.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10012689671000&title=「鬼滅の刃」の映画が台湾でも始まって大勢の人が見に行く
+      |/enqform\.html?
+    }x
+    
     # Search Engines are strict, so trigger using the default HTTP header fields
     # with +header: {}+ and fetch/set the cookie using +eat_cookie: true+.
     def initialize(url,eat_cookie: true,header: {},**kargs)
@@ -57,11 +67,8 @@ module NHKore
       link = Util.unspace_web_str(link).downcase() unless cleaned
       
       return true if link.empty?()
-      return true if link =~ /\/about\.html?/ # https://www3.nhk.or.jp/news/easy/about.html
-      return true if link =~ /\/movieplayer\.html?/ # https://www3.nhk.or.jp/news/easy/movieplayer.html?id=k10038422811_1207251719_1207251728.mp4&teacuprbbs=4feb73432045dbb97c283d64d459f7cf
-      return true if link =~ /\/audio\.html?/ # https://www3.nhk.or.jp/news/easy/player/audio.html?id=k10011555691000
-      return true if link =~ /\/news\/easy\/index\.html?/ # http://www3.nhk.or.jp/news/easy/index.html
-      return true if link =~ /cgi2.*enqform/ # https://cgi2.nhk.or.jp/news/easy/easy_enq/bin/form/enqform.html?id=k10011916321000&title=日本の会社が作った鉄道の車両「あずま」がイギリスで走る
+      
+      return true if IGNORE_LINK_REGEX.match?(link)
       
       return false
     end

data/lib/nhkore/version.rb

@@ -22,5 +22,5 @@
 
 
 module NHKore
-  VERSION = '0.3.6'
+  VERSION = '0.3.7'
 end

data/lib/nhkore/word.rb

@@ -56,11 +56,15 @@ module NHKore
       if !unknown.nil?()
         # kanji?() only tests if it contains kanji, so don't use kana?().
         if Util.kanji?(unknown)
-          raise ArgumentError,"unknown[#{unknown}] will overwrite kanji[#{kanji}]" unless Util.empty_web_str?(kanji)
+          if !Util.empty_web_str?(kanji)
+            raise ArgumentError,"unknown[#{unknown}] will overwrite kanji[#{kanji}]"
+          end
           
           kanji = unknown
         else
-          raise ArgumentError,"unknown[#{unknown}] will overwrite kana[#{kana}]" unless Util.empty_web_str?(kana)
+          if !Util.empty_web_str?(kana)
+            raise ArgumentError,"unknown[#{unknown}] will overwrite kana[#{kana}]"
+          end
           
           kana = unknown
         end

data/nhkore.gemspec

@@ -25,7 +25,6 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'nhkore/version'
 
-
 Gem::Specification.new() do |spec|
   spec.name        = 'nhkore'
   spec.version     = NHKore::VERSION
@@ -39,31 +38,33 @@ Gem::Specification.new() do |spec|
     ' Includes a CLI app and a scraper library.'
   
   spec.metadata = {
-    'bug_tracker_uri' => 'https://github.com/esotericpig/nhkore/issues',
-    'changelog_uri'   => 'https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md',
-    'homepage_uri'    => 'https://github.com/esotericpig/nhkore',
-    'source_code_uri' => 'https://github.com/esotericpig/nhkore'
+    'homepage_uri'      => 'https://github.com/esotericpig/nhkore',
+    'source_code_uri'   => 'https://github.com/esotericpig/nhkore',
+    'bug_tracker_uri'   => 'https://github.com/esotericpig/nhkore/issues',
+    'changelog_uri'     => 'https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md',
+    #'documentation_uri' => '',
+    #'wiki_uri'          => '',
+    #'mailing_list_uri'  => '',
   }
   
-  spec.require_paths = ['lib']
-  spec.bindir        = 'bin'
-  spec.executables   = [spec.name]
+  spec.requirements = [
+    'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html',
+  ]
+  
+  spec.required_ruby_version = '>= 2.4'
+  spec.require_paths         = ['lib']
+  spec.bindir                = 'bin'
+  spec.executables           = [spec.name]
   
   spec.files = [
     Dir.glob(File.join("{#{spec.require_paths.join(',')}}",'**','*.{erb,rb}')),
     Dir.glob(File.join(spec.bindir,'*')),
-    Dir.glob(File.join('{test,yard}','**','*.{erb,rb}')),
-    %W( Gemfile #{spec.name}.gemspec Rakefile ),
-    %w( CHANGELOG.md LICENSE.txt README.md ),
+    Dir.glob(File.join('{samples,test,yard}','**','*.{erb,rb}')),
+    %W[ Gemfile Gemfile.lock #{spec.name}.gemspec Rakefile .yardopts ],
+    %w[ CHANGELOG.md LICENSE.txt README.md ],
   ].flatten()
   
-  spec.required_ruby_version = '>= 2.4'
-  
-  spec.requirements = [
-    'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html',
-  ]
-  
-  spec.add_runtime_dependency 'attr_bool'            ,'~> 0.1'  # For attr_accessor?/attr_reader?
+  spec.add_runtime_dependency 'attr_bool'            ,'~> 0.2'  # For attr_accessor?/attr_reader?
   spec.add_runtime_dependency 'bimyou_segmenter'     ,'~> 1.2'  # For splitting Japanese sentences into words
   spec.add_runtime_dependency 'cri'                  ,'~> 2.15' # For CLI commands/options
   spec.add_runtime_dependency 'down'                 ,'~> 5.1'  # For downloading files (GetCmd)
@@ -97,8 +98,17 @@ Gem::Specification.new() do |spec|
   Homepage:  #{spec.homepage}
   
   Code:      #{spec.metadata['source_code_uri']}
-  Changelog: #{spec.metadata['changelog_uri']}
   Bugs:      #{spec.metadata['bug_tracker_uri']}
   
+  Changelog: #{spec.metadata['changelog_uri']}
+  
   EOM
+  
+  spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
+  
+  spec.rdoc_options = [
+    '--hyperlink-all','--show-hash',
+    '--title',"NHKore v#{NHKore::VERSION} Doc",
+    '--main','README.md',
+  ]
 end

data/samples/looper.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+# frozen_string_literal: true
+
+#--
+# This file is part of NHKore.
+# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
+# 
+# NHKore is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# NHKore is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public License
+# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+#++
+
+
+###
+# If you run this script, be aware that it uses the +-F+ force option
+# (which overwrites files without prompting).
+# 
+# @author Jonathan Bradley Whited (@esotericpig)
+# @since  0.3.2
+###
+
+case ARGV[0]
+when '-c' # count
+  system('nhkore search ez --show-count')
+  puts
+  puts %q{Use the first number with the '-a' option.}
+  exit
+when '-a' # articles
+  articles = ARGV[1].to_i()
+  articles = 0 if articles < 0
+else
+  puts 'Options:'
+  puts '  -c          show count to use with -a'
+  puts '  -a <int>    number of articles already scraped; execute scraping'
+  exit
+end
+
+articles_inc = 25
+max_errors   = 5 # Exit, for example, if 404 errors repeatedly
+max_loop     = 5 # Possible total = articles_inc * max_loop
+
+thread = Thread.new() do
+  i = 0
+  
+  while i < max_loop
+    puts "Loop #{i += 1} => #{articles} articles"
+    
+    if system("nhkore -F -t 300 -m 10 news ez -s #{articles_inc}")
+      articles += articles_inc
+    else
+      break if (max_errors -= 1) <= 0
+    end
+    
+    puts
+  end
+end
+
+# Ctrl+C
+trap('INT') do
+  if thread.alive?()
+    # Try to exit gracefully.
+    max_loop = -1
+    thread.join(5)
+    
+    # Die!
+    thread.kill() if thread.alive?()
+  end
+  
+  exit
+end
+
+thread.join() # Run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nhkore
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Jonathan Bradley Whited (@esotericpig)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attr_bool
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: bimyou_segmenter
   requirement: !ruby/object:Gem::Requirement
@@ -339,10 +339,15 @@ email:
 executables:
 - nhkore
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
 files:
+- ".yardopts"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -379,6 +384,7 @@ files:
 - lib/nhkore/version.rb
 - lib/nhkore/word.rb
 - nhkore.gemspec
+- samples/looper.rb
 - test/nhkore/test_helper.rb
 - test/nhkore_test.rb
 - yard/templates/default/layout/html/footer.erb
@@ -386,15 +392,22 @@ homepage: https://github.com/esotericpig/nhkore
 licenses:
 - LGPL-3.0-or-later
 metadata:
-  bug_tracker_uri: https://github.com/esotericpig/nhkore/issues
-  changelog_uri: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md
   homepage_uri: https://github.com/esotericpig/nhkore
   source_code_uri: https://github.com/esotericpig/nhkore
-post_install_message: "  \n  NHKore v0.3.6\n  \n  You can now use [nhkore] on the
+  bug_tracker_uri: https://github.com/esotericpig/nhkore/issues
+  changelog_uri: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md
+post_install_message: "  \n  NHKore v0.3.7\n  \n  You can now use [nhkore] on the
   command line.\n  \n  Homepage:  https://github.com/esotericpig/nhkore\n  \n  Code:
-  \     https://github.com/esotericpig/nhkore\n  Changelog: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md\n
-  \ Bugs:      https://github.com/esotericpig/nhkore/issues\n  \n"
-rdoc_options: []
+  \     https://github.com/esotericpig/nhkore\n  Bugs:      https://github.com/esotericpig/nhkore/issues\n
+  \ \n  Changelog: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md\n
+  \ \n"
+rdoc_options:
+- "--hyperlink-all"
+- "--show-hash"
+- "--title"
+- NHKore v0.3.7 Doc
+- "--main"
+- README.md
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement