nhkore 0.3.14 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5416a0276ff4d50ef49bac27caea9fc6c6629f1d7736a4429661f5c2d730d7fd
-  data.tar.gz: a82b809808ca9b0ae44db8718ea1259b261b7e9ad60292acd22697a02d0a6510
+  metadata.gz: f23192b04fc6a0c1cf225db4a029e3226346d27c4fe977ee05f1b522c40708bb
+  data.tar.gz: 3e71d5ef9eb60327cb9ced89a819de77b5cd8910e755332d5764dc95e8263f71
 SHA512:
-  metadata.gz: e1702feb47ff9db7654306169b08cbc301b198dd047e88667070dab34f4044e037fca1ca45a8e6e35aa2cb08d90fdfb22bab48456ae3a8edac8878a1fc1d89fa
-  data.tar.gz: 5893f8a805f5134a5a4fc5fd78233ae70e6196799a8278733f05ffa0cba2447cba18123b00e8a14820f30fba44ce481e03a99b0a55e0d225392865ff27008ec2
+  metadata.gz: 2cec06bd51e86ddd7b30a052b4078a2932344996bd6d97c4b7d927d0bc8fbcecbd2fb21680957551857d3a152d33409aa7ee8e1dd496a271cb31c63cbe2eb2e9
+  data.tar.gz: 186c42412e35567aebe2afee71636fb58a28510bd6ab3e8b65df6adef5373860f25e87be71e5d8499dcf6b6babb0ee3aae796a01f818732bc53ad682e584e701

data/CHANGELOG.md

@@ -5,10 +5,17 @@ All notable changes to this project will be documented in this file.
 Format is based on [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0),
 and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.14...HEAD)
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.16...HEAD)
 -
 
 
+## [v0.3.16] - 2024-08-14
+
+### Fixed
+- Fixed to work with new NHK pages.
+- Updated gems.
+
+
 ## [v0.3.14] - 2022-07-24
 
 ### Added

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    nhkore (0.3.14)
+    nhkore (0.3.16)
       attr_bool (~> 0.2)
       bimyou_segmenter (~> 1.2)
       cri (~> 2.15)
-      down (~> 5.3)
-      highline (~> 2.0)
+      down (~> 5.4)
+      highline (~> 3.1)
       http-cookie (~> 1.0)
       japanese_deinflector (~> 0.0)
-      nokogiri (~> 1.13)
+      nokogiri (~> 1.16)
       psychgus (~> 1.3)
-      public_suffix (~> 4.0)
+      public_suffix (~> 6.0)
       rainbow (~> 3.1)
-      rss (~> 0.2)
+      rss (~> 0.3)
       rubyzip (~> 2.3)
       tiny_segmenter (~> 0.0)
       tty-progressbar (~> 0.18)
@@ -22,43 +22,48 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     attr_bool (0.2.2)
     bimyou_segmenter (1.2.0)
-    cri (2.15.11)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    down (5.3.1)
+    cri (2.15.12)
+    domain_name (0.6.20240107)
+    down (5.4.2)
       addressable (~> 2.8)
-    highline (2.0.3)
-    http-cookie (1.0.5)
+    highline (3.1.0)
+      reline
+    http-cookie (1.0.6)
       domain_name (~> 0.5)
+    io-console (0.7.2)
     japanese_deinflector (0.0.2)
-    mini_portile2 (2.8.0)
-    minitest (5.16.2)
-    nokogiri (1.13.8)
-      mini_portile2 (~> 2.8.0)
+    mini_portile2 (2.8.7)
+    minitest (5.25.0)
+    nokogiri (1.16.7)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    psych (4.0.4)
+    psych (5.1.2)
       stringio
     psychgus (1.3.4)
       psych (>= 3.0)
-    public_suffix (4.0.7)
-    racc (1.6.0)
+    public_suffix (6.0.1)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.2.1)
     raketeer (0.2.13)
       rake
-    rdoc (6.4.0)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    redcarpet (3.5.1)
-    rexml (3.2.5)
-    rss (0.2.9)
+    redcarpet (3.6.0)
+    reline (0.5.9)
+      io-console (~> 0.5)
+    rexml (3.3.5)
+      strscan
+    rss (0.3.1)
       rexml
     rubyzip (2.3.2)
-    stringio (3.0.2)
+    stringio (3.1.1)
     strings-ansi (0.2.0)
+    strscan (3.1.0)
     tiny_segmenter (0.0.6)
     tty-cursor (0.7.1)
     tty-progressbar (0.18.2)
@@ -66,16 +71,11 @@ GEM
       tty-cursor (~> 0.7)
       tty-screen (~> 0.8)
       unicode-display_width (>= 1.6, < 3.0)
-    tty-screen (0.8.1)
+    tty-screen (0.8.2)
     tty-spinner (0.9.3)
       tty-cursor (~> 0.7)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.2)
-    unicode-display_width (2.2.0)
-    webrick (1.7.0)
-    yard (0.9.28)
-      webrick (~> 1.7.0)
+    unicode-display_width (2.5.0)
+    yard (0.9.36)
     yard_ghurt (1.2.1)
       rake
       yard
@@ -84,15 +84,15 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 2.3)
-  minitest (~> 5.16)
+  bundler (~> 2.5)
+  minitest (~> 5.25)
   nhkore!
-  rake (~> 13.0)
+  rake (~> 13.2)
   raketeer (~> 0.2)
-  rdoc (~> 6.4)
-  redcarpet (~> 3.5)
+  rdoc (~> 6.7)
+  redcarpet (~> 3.6)
   yard (~> 0.9)
   yard_ghurt (~> 1.2)
 
 BUNDLED WITH
-   2.3.18
+   2.5.17

data/README.md

@@ -867,12 +867,11 @@ This will update *core/* for you:
 2. Update *core* package:
     - `$ bundle exec rake update_core`
     - `$ bundle exec rake clobber pkg_core`
-3. Create a new tag & release:
+3. Commit & Push.
+4. Create a new tag & release:
     - Note: make sure to add *pkg/nhkore-core.zip*
     - `$ gh release create v0 pkg/*.gem pkg/*.zip`
-    - `$ git pull`
-4. Release to *GitHub Packages*:
-    - With *Raketary*: `$ raketary github_pkg`
+    - `$ git pull && git fetch`
 5. Release to *RubyGems*:
     - `$ bundle exec rake release`
 

data/Rakefile

@@ -56,6 +56,9 @@ task :update_core do |task|
   cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
   hl = HighLine.new
 
+  next unless sh(*cmd,'se','--show-count','ez')
+  puts
+
   next unless sh(*cmd,'se','-l','10','ez','bing')
   next unless hl.agree(continue_msg)
   puts
@@ -74,7 +77,6 @@ task :update_core do |task|
   puts
 end
 
-# @since 0.3.6
 desc 'Update showcase file for release'
 task :update_showcase do |task|
   require 'highline'

data/lib/nhkore/app.rb

@@ -27,18 +27,11 @@ require 'nhkore/cli/sift_cmd'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module CLI
   end
 
   ###
   # For disabling/enabling color output.
-  #
-  # @author Jonathan Bradley Whited
-  # @since  0.2.1
   ###
   module CriColorExt
     @color = true
@@ -52,10 +45,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class App
     include CLI::FXCmd
     include CLI::GetCmd
@@ -576,10 +565,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class NoProgressBar
     MSG = '%{title}... %{percent}%%'
     PUT_INTERVAL = 100.0 / 6.25
@@ -627,7 +612,7 @@ module NHKore
 
       @tokens[:advance] = percent
 
-      puts to_s
+      puts self
     end
 
     def finish
@@ -635,7 +620,7 @@ module NHKore
     end
 
     def start
-      puts to_s
+      puts self
     end
 
     def to_s

data/lib/nhkore/article.rb

@@ -16,10 +16,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Article
     attr_reader :datetime
     attr_reader :futsuurl
@@ -29,7 +25,7 @@ module NHKore
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @datetime = nil
       @futsuurl = nil
@@ -101,13 +97,13 @@ module NHKore
     end
 
     def futsuurl=(value)
-      # Don't store URI, store String.
-      @futsuurl = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @futsuurl = value&.to_s
     end
 
     def url=(value)
-      # Don't store URI, store String.
-      @url = value.nil? ? nil : value.to_s
+      # Don't store URI, store String or nil.
+      @url = value&.to_s
     end
 
     def to_s(mini: false)

data/lib/nhkore/article_scraper.rb

@@ -26,10 +26,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class ArticleScraper < Scraper
     extend AttrBool::Ext
 
@@ -139,7 +135,13 @@ module NHKore
     end
 
     def parse_dicwin_id(str)
-      str = str.gsub(/\D+/,'')
+      str = str.to_s.strip.downcase
+
+      if str.start_with?('id-') # 'id-0000'
+        str = str.gsub(/\D+/,'')
+      else # 'RSHOK-K-003806'
+        # Same.
+      end
 
       return nil if str.empty?
       return str
@@ -235,8 +237,6 @@ module NHKore
           # Ignore; try again below.
           Util.warn("could not parse date time[#{tag_text}] from tag[#{tag_name}] at URL[#{@url}]: #{e}")
         end
-
-        return datetime
       end
 
       # Third, try body's id.
@@ -393,7 +393,6 @@ module NHKore
       return link
     end
 
-    # @since 0.3.8
     # @see https://www3.nhk.or.jp/news/easy/k10012759201000/k10012759201000.html
     def scrape_ruby_words(tag,result: ScrapeWordsResult.new)
       words = Word.scrape_ruby_tag(tag,missingno: @missingno,url: @url)
@@ -489,15 +488,21 @@ module NHKore
     end
 
     def scrape_title(doc,article)
+      # Not grabbing `<head><title>` because it doesn't have `<ruby>` tags.
+
       tag = doc.css('h1.article-main__title')
       tag_name = nil
 
+      if tag.length < 1
+        # - https://www3.nhk.or.jp/news/easy/em2024081312029/em2024081312029.html
+        tag = doc.css('h1.article-title') # No warning.
+      end
+
       if tag.length < 1
         # - https://www3.nhk.or.jp/news/easy/article/disaster_earthquake_illust.html
         tag_name = 'h1.article-eq__title'
         tag = doc.css(tag_name)
       end
-
       if tag.length < 1 && !@strict
         # This shouldn't be used except for select sites.
         # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
@@ -632,16 +637,12 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class ScrapeWordsResult
     attr_reader :text
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @text = ''.dup
       @words = []

data/lib/nhkore/cleaner.rb

@@ -14,10 +14,6 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Cleaner
     def begin_clean(str)
       return str
@@ -53,10 +49,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BasicCleaner < Cleaner
     def end_clean(str)
       # This is very simple, as Splitter will split on punctuation,
@@ -70,10 +62,6 @@ module NHKore
     end
   end
 
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class BestCleaner < BasicCleaner
   end
 end

data/lib/nhkore/cli/fx_cmd.rb

@@ -11,10 +11,6 @@
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module FXCmd
     def build_fx_cmd
       app = self

data/lib/nhkore/cli/get_cmd.rb

@@ -14,10 +14,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module GetCmd
     DEFAULT_GET_CHUNK_SIZE = 4 * 1024
     DEFAULT_GET_URL_LENGTH = 11_000_000 # Just a generous estimation used as a fallback; may be outdated.

data/lib/nhkore/cli/news_cmd.rb

@@ -21,10 +21,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module NewsCmd
     DEFAULT_NEWS_SCRAPE = 1
 
@@ -255,16 +251,20 @@ module CLI
           next if !redo_scrapes && scraped_news_article?(news,link)
 
           url = link.url
+          result = scrape_news_article(url,link: link,new_articles: new_articles,news: news)
 
-          if (new_url = scrape_news_article(url,link: link,new_articles: new_articles,news: news))
+          if result == :scraped
+            scrape_count += 1
+          elsif result == :unscraped
+            next
+          else
             # --show-dict
-            url = new_url
-            scrape_count = max_scrapes - 1 # Break on next iteration for update_spin_detail()
+            url = result
+            scrape_count = max_scrapes # Break on next iteration for update_spin_detail().
           end
 
           # Break on next iteration for update_spin_detail().
-          next if (scrape_count += 1) >= max_scrapes
-
+          next if scrape_count >= max_scrapes
           sleep_scraper
         end
       else
@@ -275,9 +275,8 @@ module CLI
           links.add_link(link)
         end
 
-        scrape_news_article(url,link: link,new_articles: new_articles,news: news)
-
-        scrape_count += 1
+        result = scrape_news_article(url,link: link,new_articles: new_articles,news: news)
+        scrape_count += 1 if result != :unscraped
       end
 
       stop_spin
@@ -338,9 +337,17 @@ module CLI
         return scraper.url
       end
 
-      scraper = ArticleScraper.new(url,**@news_article_scraper_kargs)
-      article = scraper.scrape
+      scraper = nil
+
+      begin
+        scraper = ArticleScraper.new(url,**@news_article_scraper_kargs)
+      rescue Http404Error
+        # - https://www3.nhk.or.jp/news/easy/k10014157491000/k10014157491000.html
+        Util.warn("Ignoring URL with 404 error: #{url}.")
+        return :unscraped
+      end
 
+      article = scraper.scrape
       # run_news_cmd() handles overwriting with --redo or not
       #   using scraped_news_article?().
       news.add_article(article,overwrite: true)
@@ -350,7 +357,7 @@ module CLI
 
       new_articles << article
 
-      return false # No --show-dict
+      return :scraped # No --show-dict
     end
 
     def scraped_news_article?(news,link)
@@ -366,10 +373,15 @@ module CLI
         end
 
         if article.nil?
-          scraper = ArticleScraper.new(link.url,**@news_article_scraper_kargs)
+          scraper = nil
 
-          sha256 = scraper.scrape_sha256_only
+          begin
+            scraper = ArticleScraper.new(link.url,**@news_article_scraper_kargs)
+          rescue Http404Error
+            return false
+          end
 
+          sha256 = scraper.scrape_sha256_only
           article = news.article_with_sha256(sha256) if news.sha256?(sha256)
         end
       end

data/lib/nhkore/cli/search_cmd.rb

@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.3.0
-  ###
   module SearchCmd
     def build_search_cmd
       app = self

data/lib/nhkore/cli/sift_cmd.rb

@@ -20,10 +20,6 @@ require 'nhkore/util'
 
 module NHKore
 module CLI
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   module SiftCmd
     DEFAULT_SIFT_EXT = :csv
     DEFAULT_SIFT_FUTSUU_FILE = "#{Sifter::DEFAULT_FUTSUU_FILE}{search.criteria}{file.ext}"
@@ -260,7 +256,7 @@ module CLI
       puts
 
       if dry_run
-        puts sifter.to_s
+        puts sifter
       else
         start_spin('Saving sifted data to file')
 

data/lib/nhkore/datetime_parser.rb

@@ -17,10 +17,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.3.4
-  ###
   class DatetimeParser
     extend AttrBool::Ext
 
@@ -181,7 +177,7 @@ module NHKore
       return self if @min_or_max
 
       has_small = false
-      jst_now = Util.jst_now()
+      jst_now = Util.jst_now
 
       # Must be from smallest to biggest.
 

data/lib/nhkore/defn.rb

@@ -16,17 +16,13 @@ require 'nhkore/word'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Defn
     attr_reader :hyoukis
     attr_accessor :text
     attr_reader :words
 
     def initialize
-      super()
+      super
 
       @hyoukis = []
       @text = ''.dup

data/lib/nhkore/dict.rb

@@ -14,15 +14,11 @@ require 'nhkore/error'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Dict
     attr_reader :entries
 
     def initialize
-      super()
+      super
 
       @entries = {}
     end
@@ -39,6 +35,7 @@ module NHKore
       dict = Dict.new
 
       hash.each do |id,array|
+        id = id.to_s.strip.downcase # 'RSHOK-K-003806', '0000'
         entry = Entry.scrape(id,array,missingno: missingno,url: url)
 
         next if entry.nil?

data/lib/nhkore/dict_scraper.rb

@@ -16,10 +16,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class DictScraper < Scraper
     attr_accessor :missingno
 

data/lib/nhkore/entry.rb

@@ -14,10 +14,6 @@ require 'nhkore/util'
 
 
 module NHKore
-  ###
-  # @author Jonathan Bradley Whited
-  # @since  0.2.0
-  ###
   class Entry
     HYOUKI_SEP = '・'
 
@@ -25,18 +21,16 @@ module NHKore
     attr_accessor :id
 
     def initialize
-      super()
+      super
 
       @defns = []
       @id = nil
     end
 
     def build_defn
-      defns = []
       i = 0
-
-      @defns.each do |defn|
-        defns << "#{i += 1}）#{defn}" # Japanese parenthesis
+      defns = @defns.map do |defn|
+        "#{i += 1}）#{defn}" # Japanese parenthesis
       end
 
       return defns.join("\n")