nhkore 0.3.13 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21663a8ce4850b7f03361289832cfdb6caa5bc64e62af1ff7cd4b91b7fc2329b
-  data.tar.gz: de45874fc3834c492ea74ad26067504629185db39fff2f42148a24b4be453cc9
+  metadata.gz: f23192b04fc6a0c1cf225db4a029e3226346d27c4fe977ee05f1b522c40708bb
+  data.tar.gz: 3e71d5ef9eb60327cb9ced89a819de77b5cd8910e755332d5764dc95e8263f71
 SHA512:
-  metadata.gz: 0ea9413c534cb11d60764e6dd95473b65e6a76418b28ca69441c18741cd6920992f0289a6d9f5c39ed0124023296992f7a60696099ce056b009cadcf8e863867
-  data.tar.gz: a223dd3e9416b5487274e3218fe55e6c936019a4faf58343a6c4fc9a3bc69c55805c08f42411337ce672766cd72934c423aff734bce766273e9a5707d356eceb
+  metadata.gz: 2cec06bd51e86ddd7b30a052b4078a2932344996bd6d97c4b7d927d0bc8fbcecbd2fb21680957551857d3a152d33409aa7ee8e1dd496a271cb31c63cbe2eb2e9
+  data.tar.gz: 186c42412e35567aebe2afee71636fb58a28510bd6ab3e8b65df6adef5373860f25e87be71e5d8499dcf6b6babb0ee3aae796a01f818732bc53ad682e584e701

data/CHANGELOG.md

@@ -5,10 +5,26 @@ All notable changes to this project will be documented in this file.
 Format is based on [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0),
 and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.13...HEAD)
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.16...HEAD)
 -
 
 
+## [v0.3.16] - 2024-08-14
+
+### Fixed
+- Fixed to work with new NHK pages.
+- Updated gems.
+
+
+## [v0.3.14] - 2022-07-24
+
+### Added
+- `--loop` option to `search` command so can run web search (search engine) multiple times since this usually doesn't get all results if only do once.
+
+### Fixed
+- Updated gems (`nokogiri`).
+
+
 ## [v0.3.13] - 2022-04-27
 
 ### Fixed

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    nhkore (0.3.13)
+    nhkore (0.3.16)
       attr_bool (~> 0.2)
       bimyou_segmenter (~> 1.2)
       cri (~> 2.15)
-      down (~> 5.3)
-      highline (~> 2.0)
+      down (~> 5.4)
+      highline (~> 3.1)
       http-cookie (~> 1.0)
       japanese_deinflector (~> 0.0)
-      nokogiri (~> 1.13)
+      nokogiri (~> 1.16)
       psychgus (~> 1.3)
-      public_suffix (~> 4.0)
+      public_suffix (~> 6.0)
       rainbow (~> 3.1)
-      rss (~> 0.2)
+      rss (~> 0.3)
       rubyzip (~> 2.3)
       tiny_segmenter (~> 0.0)
       tty-progressbar (~> 0.18)
@@ -22,43 +22,48 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     attr_bool (0.2.2)
     bimyou_segmenter (1.2.0)
-    cri (2.15.11)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    down (5.3.1)
+    cri (2.15.12)
+    domain_name (0.6.20240107)
+    down (5.4.2)
       addressable (~> 2.8)
-    highline (2.0.3)
-    http-cookie (1.0.4)
+    highline (3.1.0)
+      reline
+    http-cookie (1.0.6)
       domain_name (~> 0.5)
+    io-console (0.7.2)
     japanese_deinflector (0.0.2)
-    mini_portile2 (2.8.0)
-    minitest (5.15.0)
-    nokogiri (1.13.4)
-      mini_portile2 (~> 2.8.0)
+    mini_portile2 (2.8.7)
+    minitest (5.25.0)
+    nokogiri (1.16.7)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    psych (4.0.3)
+    psych (5.1.2)
       stringio
     psychgus (1.3.4)
       psych (>= 3.0)
-    public_suffix (4.0.7)
-    racc (1.6.0)
+    public_suffix (6.0.1)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.2.1)
     raketeer (0.2.13)
       rake
-    rdoc (6.4.0)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    redcarpet (3.5.1)
-    rexml (3.2.5)
-    rss (0.2.9)
+    redcarpet (3.6.0)
+    reline (0.5.9)
+      io-console (~> 0.5)
+    rexml (3.3.5)
+      strscan
+    rss (0.3.1)
       rexml
     rubyzip (2.3.2)
-    stringio (3.0.1)
+    stringio (3.1.1)
     strings-ansi (0.2.0)
+    strscan (3.1.0)
     tiny_segmenter (0.0.6)
     tty-cursor (0.7.1)
     tty-progressbar (0.18.2)
@@ -66,16 +71,11 @@ GEM
       tty-cursor (~> 0.7)
       tty-screen (~> 0.8)
       unicode-display_width (>= 1.6, < 3.0)
-    tty-screen (0.8.1)
+    tty-screen (0.8.2)
     tty-spinner (0.9.3)
       tty-cursor (~> 0.7)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.1)
-    unicode-display_width (2.1.0)
-    webrick (1.7.0)
-    yard (0.9.27)
-      webrick (~> 1.7.0)
+    unicode-display_width (2.5.0)
+    yard (0.9.36)
     yard_ghurt (1.2.1)
       rake
       yard
@@ -84,15 +84,15 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 2.3)
-  minitest (~> 5.15)
+  bundler (~> 2.5)
+  minitest (~> 5.25)
   nhkore!
-  rake (~> 13.0)
+  rake (~> 13.2)
   raketeer (~> 0.2)
-  rdoc (~> 6.4)
-  redcarpet (~> 3.5)
+  rdoc (~> 6.7)
+  redcarpet (~> 3.6)
   yard (~> 0.9)
   yard_ghurt (~> 1.2)
 
 BUNDLED WITH
-   2.3.12
+   2.5.17

data/README.md

@@ -867,12 +867,11 @@ This will update *core/* for you:
 2. Update *core* package:
     - `$ bundle exec rake update_core`
     - `$ bundle exec rake clobber pkg_core`
-3. Create a new tag & release:
+3. Commit & Push.
+4. Create a new tag & release:
     - Note: make sure to add *pkg/nhkore-core.zip*
     - `$ gh release create v0 pkg/*.gem pkg/*.zip`
-    - `$ git pull`
-4. Release to *GitHub Packages*:
-    - With *Raketary*: `$ raketary github_pkg`
+    - `$ git pull && git fetch`
 5. Release to *RubyGems*:
     - `$ bundle exec rake release`
 
@@ -885,7 +884,7 @@ Releasing new HTML file for website:
 [GNU LGPL v3+](LICENSE.txt)
 
 > NHKore (<https://github.com/esotericpig/nhkore>)  
-> Copyright (c) 2020-2021 Jonathan Bradley Whited  
+> Copyright (c) 2020-2022 Jonathan Bradley Whited  
 > 
 > NHKore is free software: you can redistribute it and/or modify  
 > it under the terms of the GNU Lesser General Public License as published by  

data/Rakefile

@@ -56,11 +56,14 @@ task :update_core do |task|
   cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
   hl = HighLine.new
 
-  next unless sh(*cmd,'se','ez','bing')
+  next unless sh(*cmd,'se','--show-count','ez')
+  puts
+
+  next unless sh(*cmd,'se','-l','10','ez','bing')
   next unless hl.agree(continue_msg)
   puts
 
-  next unless sh(*cmd,'news','-s','500','ez')
+  next unless sh(*cmd,'news','-s','1000','ez')
   next unless hl.agree(continue_msg)
   puts
 
@@ -74,7 +77,6 @@ task :update_core do |task|
   puts
 end
 
-# @since 0.3.6
 desc 'Update showcase file for release'
 task :update_showcase do |task|
   require 'highline'

data/lib/nhkore/app.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -27,18 +27,11 @@ require 'nhkore/cli/sift_cmd'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module CLI
   end
 
   ###
   # For disabling/enabling color output.
-  #
-  # @author Jonathan Bradley Whited
-  # @since  0.2.1
   ###
   module CriColorExt
     @color = true
@@ -52,10 +45,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class App
     include CLI::FXCmd
     include CLI::GetCmd
@@ -537,7 +526,11 @@ module NHKore
     end
 
     def sleep_scraper
-      sleep(@sleep_time)
+      # Do a range to better emulate being a human.
+      r = rand(@sleep_time..(@sleep_time + 0.1111))
+      s = r.round(3) # Within 1000ms (0.000 - 0.999).
+
+      sleep(s)
     end
 
     def start_spin(title,detail: '')
@@ -572,10 +565,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class NoProgressBar
     MSG = '%{title}... %{percent}%%'
     PUT_INTERVAL = 100.0 / 6.25
@@ -623,7 +612,7 @@ module NHKore
 
       @tokens[:advance] = percent
 
-      puts to_s
+      puts self
     end
 
     def finish
@@ -631,7 +620,7 @@ module NHKore
     end
 
     def start
-      puts to_s
+      puts self
     end
 
     def to_s

data/lib/nhkore/article.rb

@@ -16,10 +16,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Article
     attr_reader :datetime
     attr_reader :futsuurl
@@ -29,7 +25,7 @@ module NHKore
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @datetime = nil
       @futsuurl = nil
@@ -101,13 +97,13 @@ module NHKore
     end
 
     def futsuurl=(value)
-      # Don't store URI, store String.
-      @futsuurl = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @futsuurl = value&.to_s
     end
 
     def url=(value)
-      # Don't store URI, store String.
-      @url = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @url = value&.to_s
     end
 
     def to_s(mini: false)

data/lib/nhkore/article_scraper.rb

@@ -26,10 +26,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class ArticleScraper < Scraper
     extend AttrBool::Ext
 
@@ -139,7 +135,13 @@ module NHKore
     end
 
     def parse_dicwin_id(str)
-      str = str.gsub(/\D+/,'')
+      str = str.to_s.strip.downcase
+
+      if str.start_with?('id-') # 'id-0000'
+        str = str.gsub(/\D+/,'')
+      else # 'RSHOK-K-003806'
+        # Same.
+      end
 
       return nil if str.empty?
       return str
@@ -235,8 +237,6 @@ module NHKore
           # Ignore; try again below.
           Util.warn("could not parse date time[#{tag_text}] from tag[#{tag_name}] at URL[#{@url}]: #{e}")
         end
-
-        return datetime
       end
 
       # Third, try body's id.
@@ -393,7 +393,6 @@ module NHKore
       return link
     end
 
-    # @since 0.3.8
     # @see https://www3.nhk.or.jp/news/easy/k10012759201000/k10012759201000.html
     def scrape_ruby_words(tag,result: ScrapeWordsResult.new)
       words = Word.scrape_ruby_tag(tag,missingno: @missingno,url: @url)
@@ -489,15 +488,21 @@ module NHKore
     end
 
     def scrape_title(doc,article)
+      # Not grabbing `<head><title>` because it doesn't have `<ruby>` tags.
+
       tag = doc.css('h1.article-main__title')
       tag_name = nil
 
+      if tag.length < 1
+        # - https://www3.nhk.or.jp/news/easy/em2024081312029/em2024081312029.html
+        tag = doc.css('h1.article-title') # No warning.
+      end
+
       if tag.length < 1
         # - https://www3.nhk.or.jp/news/easy/article/disaster_earthquake_illust.html
         tag_name = 'h1.article-eq__title'
         tag = doc.css(tag_name)
       end
-
       if tag.length < 1 && !@strict
         # This shouldn't be used except for select sites.
         # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
@@ -632,16 +637,12 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class ScrapeWordsResult
     attr_reader :text
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @text = ''.dup
       @words = []

data/lib/nhkore/cleaner.rb

@@ -14,10 +14,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Cleaner
     def begin_clean(str)
       return str
@@ -53,10 +49,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicCleaner < Cleaner
     def end_clean(str)
       # This is very simple, as Splitter will split on punctuation,
@@ -70,10 +62,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestCleaner < BasicCleaner
   end
 end

data/lib/nhkore/cli/fx_cmd.rb

@@ -11,10 +11,6 @@
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module FXCmd
     def build_fx_cmd
       app = self

data/lib/nhkore/cli/get_cmd.rb

@@ -14,10 +14,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module GetCmd
     DEFAULT_GET_CHUNK_SIZE = 4 * 1024
     DEFAULT_GET_URL_LENGTH = 11_000_000 # Just a generous estimation used as a fallback; may be outdated.

data/lib/nhkore/cli/news_cmd.rb

@@ -21,10 +21,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module NewsCmd
     DEFAULT_NEWS_SCRAPE = 1
 
@@ -255,16 +251,20 @@ module CLI
           next if !redo_scrapes && scraped_news_article?(news,link)
 
           url = link.url
+          result = scrape_news_article(url,link: link,new_articles: new_articles,news: news)
 
-          if (new_url = scrape_news_article(url,link: link,new_articles: new_articles,news: news))
+          if result == :scraped
+            scrape_count += 1
+          elsif result == :unscraped
+            next
+          else
             # --show-dict
-            url = new_url
-            scrape_count = max_scrapes - 1 # Break on next iteration for update_spin_detail()
+            url = result
+            scrape_count = max_scrapes # Break on next iteration for update_spin_detail().
           end
 
           # Break on next iteration for update_spin_detail().
-          next if (scrape_count += 1) >= max_scrapes
-
+          next if scrape_count >= max_scrapes
           sleep_scraper
         end
       else
@@ -275,9 +275,8 @@ module CLI
           links.add_link(link)
         end
 
-        scrape_news_article(url,link: link,new_articles: new_articles,news: news)
-
-        scrape_count += 1
+        result = scrape_news_article(url,link: link,new_articles: new_articles,news: news)
+        scrape_count += 1 if result != :unscraped
       end
 
       stop_spin
@@ -338,9 +337,17 @@ module CLI
         return scraper.url
       end
 
-      scraper = ArticleScraper.new(url,**@news_article_scraper_kargs)
-      article = scraper.scrape
+      scraper = nil
+
+      begin
+        scraper = ArticleScraper.new(url,**@news_article_scraper_kargs)
+      rescue Http404Error
+        # - https://www3.nhk.or.jp/news/easy/k10014157491000/k10014157491000.html
+        Util.warn("Ignoring URL with 404 error: #{url}.")
+        return :unscraped
+      end
 
+      article = scraper.scrape
       # run_news_cmd() handles overwriting with --redo or not
       #   using scraped_news_article?().
       news.add_article(article,overwrite: true)
@@ -350,7 +357,7 @@ module CLI
 
       new_articles << article
 
-      return false # No --show-dict
+      return :scraped # No --show-dict
     end
 
     def scraped_news_article?(news,link)
@@ -366,10 +373,15 @@ module CLI
         end
 
         if article.nil?
-          scraper = ArticleScraper.new(link.url,**@news_article_scraper_kargs)
+          scraper = nil
 
-          sha256 = scraper.scrape_sha256_only
+          begin
+            scraper = ArticleScraper.new(link.url,**@news_article_scraper_kargs)
+          rescue Http404Error
+            return false
+          end
 
+          sha256 = scraper.scrape_sha256_only
           article = news.article_with_sha256(sha256) if news.sha256?(sha256)
         end
       end