nhkore 0.3.13 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21663a8ce4850b7f03361289832cfdb6caa5bc64e62af1ff7cd4b91b7fc2329b
-  data.tar.gz: de45874fc3834c492ea74ad26067504629185db39fff2f42148a24b4be453cc9
+  metadata.gz: 5416a0276ff4d50ef49bac27caea9fc6c6629f1d7736a4429661f5c2d730d7fd
+  data.tar.gz: a82b809808ca9b0ae44db8718ea1259b261b7e9ad60292acd22697a02d0a6510
 SHA512:
-  metadata.gz: 0ea9413c534cb11d60764e6dd95473b65e6a76418b28ca69441c18741cd6920992f0289a6d9f5c39ed0124023296992f7a60696099ce056b009cadcf8e863867
-  data.tar.gz: a223dd3e9416b5487274e3218fe55e6c936019a4faf58343a6c4fc9a3bc69c55805c08f42411337ce672766cd72934c423aff734bce766273e9a5707d356eceb
+  metadata.gz: e1702feb47ff9db7654306169b08cbc301b198dd047e88667070dab34f4044e037fca1ca45a8e6e35aa2cb08d90fdfb22bab48456ae3a8edac8878a1fc1d89fa
+  data.tar.gz: 5893f8a805f5134a5a4fc5fd78233ae70e6196799a8278733f05ffa0cba2447cba18123b00e8a14820f30fba44ce481e03a99b0a55e0d225392865ff27008ec2

data/CHANGELOG.md

@@ -5,10 +5,19 @@ All notable changes to this project will be documented in this file.
 Format is based on [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0),
 and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.13...HEAD)
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.14...HEAD)
 -
 
 
+## [v0.3.14] - 2022-07-24
+
+### Added
+- `--loop` option to `search` command so can run web search (search engine) multiple times since this usually doesn't get all results if only do once.
+
+### Fixed
+- Updated gems (`nokogiri`).
+
+
 ## [v0.3.13] - 2022-04-27
 
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nhkore (0.3.13)
+    nhkore (0.3.14)
       attr_bool (~> 0.2)
       bimyou_segmenter (~> 1.2)
       cri (~> 2.15)
@@ -32,15 +32,15 @@ GEM
     down (5.3.1)
       addressable (~> 2.8)
     highline (2.0.3)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     japanese_deinflector (0.0.2)
     mini_portile2 (2.8.0)
-    minitest (5.15.0)
-    nokogiri (1.13.4)
+    minitest (5.16.2)
+    nokogiri (1.13.8)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    psych (4.0.3)
+    psych (4.0.4)
       stringio
     psychgus (1.3.4)
       psych (>= 3.0)
@@ -57,7 +57,7 @@ GEM
     rss (0.2.9)
       rexml
     rubyzip (2.3.2)
-    stringio (3.0.1)
+    stringio (3.0.2)
     strings-ansi (0.2.0)
     tiny_segmenter (0.0.6)
     tty-cursor (0.7.1)
@@ -71,10 +71,10 @@ GEM
       tty-cursor (~> 0.7)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.1)
-    unicode-display_width (2.1.0)
+    unf_ext (0.0.8.2)
+    unicode-display_width (2.2.0)
     webrick (1.7.0)
-    yard (0.9.27)
+    yard (0.9.28)
       webrick (~> 1.7.0)
     yard_ghurt (1.2.1)
       rake
@@ -85,7 +85,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 2.3)
-  minitest (~> 5.15)
+  minitest (~> 5.16)
   nhkore!
   rake (~> 13.0)
   raketeer (~> 0.2)
@@ -95,4 +95,4 @@ DEPENDENCIES
   yard_ghurt (~> 1.2)
 
 BUNDLED WITH
-   2.3.12
+   2.3.18

data/README.md

@@ -885,7 +885,7 @@ Releasing new HTML file for website:
 [GNU LGPL v3+](LICENSE.txt)
 
 > NHKore (<https://github.com/esotericpig/nhkore>)  
-> Copyright (c) 2020-2021 Jonathan Bradley Whited  
+> Copyright (c) 2020-2022 Jonathan Bradley Whited  
 > 
 > NHKore is free software: you can redistribute it and/or modify  
 > it under the terms of the GNU Lesser General Public License as published by  

data/Rakefile

@@ -56,11 +56,11 @@ task :update_core do |task|
   cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
   hl = HighLine.new
 
-  next unless sh(*cmd,'se','ez','bing')
+  next unless sh(*cmd,'se','-l','10','ez','bing')
   next unless hl.agree(continue_msg)
   puts
 
-  next unless sh(*cmd,'news','-s','500','ez')
+  next unless sh(*cmd,'news','-s','1000','ez')
   next unless hl.agree(continue_msg)
   puts
 

data/lib/nhkore/app.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -537,7 +537,11 @@ module NHKore
     end
 
     def sleep_scraper
-      sleep(@sleep_time)
+      # Do a range to better emulate being a human.
+      r = rand(@sleep_time..(@sleep_time + 0.1111))
+      s = r.round(3) # Within 1000ms (0.000 - 0.999).
+
+      sleep(s)
     end
 
     def start_spin(title,detail: '')

data/lib/nhkore/cli/search_cmd.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -42,6 +42,12 @@ module CLI
         DESC
           app.check_empty_opt(:in,value)
         }
+        option :l,:loop,'number of times to repeat the search to ensure results',argument: :required,
+          transform: lambda { |value|
+            value = value.to_i
+            value = 1 if value < 1
+            value
+          }
         option :o,:out,<<-DESC,argument: :required,transform: lambda { |value|
           'directory/file' to save links to; if you only specify a directory or a file, it will attach the
           appropriate default directory/file name
@@ -164,6 +170,8 @@ module CLI
 
       dry_run = @cmd_opts[:dry_run]
       in_file = @cmd_opts[:in]
+      loop_times = @cmd_opts[:loop]
+      loop_times = 1 if loop_times.nil? || loop_times < 1
       out_file = @cmd_opts[:out]
       result_count = @cmd_opts[:results]
       result_count = SearchScraper::DEFAULT_RESULT_COUNT if result_count.nil?
@@ -174,9 +182,6 @@ module CLI
       is_file = !in_file.nil?
       links = nil
       new_links = [] # For --dry-run
-      next_page = NextPage.new
-      page_count = 0
-      page_num = 1
       url = in_file # nil will use default URL, else a file
 
       # Load previous links for 'scraped?' vars.
@@ -196,43 +201,52 @@ module CLI
         end
 
         puts "#{scraped_count} of #{links_count} links scraped."
-
         return
       end
 
-      range = (0..10_000) # Do a range to prevent an infinite loop; ichiman!
-
-      case search_type
-      # Anything that extends SearchScraper.
-      when :bing
-        range.each do
-          scraper = nil
-
-          case search_type
-          when :bing
-            scraper = BingScraper.new(nhk_type,count: result_count,is_file: is_file,url: url,**@scraper_kargs)
-          else
-            raise NHKore::Error,"internal code broken; add missing search_type[#{search_type}]"
-          end
+      1.upto(loop_times) do |loop_i|
+        page_range = (0..10_000) # Do a range to prevent an infinite loop; ichiman!
+
+        next_page = NextPage.new
+        page_count = 0
+        page_num = 1
+
+        case search_type
+        # Anything that extends SearchScraper.
+        when :bing
+          page_range.each do
+            scraper = nil
+
+            case search_type
+            when :bing
+              scraper = BingScraper.new(
+                nhk_type,count: result_count,is_file: is_file,url: url,**@scraper_kargs
+              )
+            else
+              raise NHKore::Error,"internal code broken; add missing search_type[#{search_type}]"
+            end
 
-          next_page = scraper.scrape(links,next_page)
+            next_page = scraper.scrape(links,next_page)
 
-          new_links.concat(links.links.values[links_count..-1])
-          links_count = links.length
-          page_count = next_page.count if next_page.count > 0
+            new_links.concat(links.links.values[links_count..])
+            links_count = links.length
+            page_count = next_page.count if next_page.count > 0
 
-          update_spin_detail(" (page=#{page_num}, count=#{page_count}, links=#{links.length}," \
-            " new_links=#{new_links.length})")
+            update_spin_detail(
+              format(' (%d/%d, page=%d, count=%d, links=%d, new_links=%d)',
+                     loop_i,loop_times,page_num,page_count,links.length,new_links.length)
+            )
 
-          break if next_page.empty?
+            break if next_page.empty?
 
-          page_num += 1
-          url = next_page.url
+            page_num += 1
+            url = next_page.url
 
-          sleep_scraper
+            sleep_scraper
+          end
+        else
+          raise ArgumentError,"invalid search_type[#{search_type}]"
         end
-      else
-        raise ArgumentError,"invalid search_type[#{search_type}]"
       end
 
       stop_spin

data/lib/nhkore/dict_scraper.rb

@@ -3,7 +3,7 @@
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020-2021 Jonathan Bradley Whited
+# Copyright (c) 2020-2022 Jonathan Bradley Whited
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 #++
@@ -39,7 +39,7 @@ module NHKore
       i = url.rindex(%r{[/\\]}) # Can be a URL or a file
       i = i.nil? ? 0 : (i + 1) # If no match found, no path
 
-      basename = File.basename(url[i..-1],'.*') if basename.nil?
+      basename = File.basename(url[i..],'.*') if basename.nil?
       path = url[0...i]
 
       return "#{path}#{basename}.out.dic"

data/lib/nhkore/version.rb

@@ -10,5 +10,5 @@
 
 
 module NHKore
-  VERSION = '0.3.13'
+  VERSION = '0.3.14'
 end

data/nhkore.gemspec

@@ -26,48 +26,57 @@ Gem::Specification.new do |spec|
     #'mailing_list_uri'  => '',
   }
 
+  spec.required_ruby_version = '>= 2.5'
   spec.requirements = [
     'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html',
   ]
 
-  spec.required_ruby_version = '>= 2.5'
-  spec.require_paths         = ['lib']
-  spec.bindir                = 'bin'
-  spec.executables           = [spec.name]
+  spec.require_paths = ['lib']
+  spec.bindir        = 'bin'
+  spec.executables   = [spec.name]
+
+  spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
+  spec.rdoc_options = [
+    '--hyperlink-all','--show-hash',
+    '--title',"NHKore v#{NHKore::VERSION} Doc",
+    '--main','README.md',
+  ]
 
   spec.files = [
     Dir.glob(File.join("{#{spec.require_paths.join(',')}}",'**','*.{erb,rb}')),
     Dir.glob(File.join(spec.bindir,'*')),
     Dir.glob(File.join('{samples,test,yard}','**','*.{erb,rb}')),
     %W[ Gemfile Gemfile.lock #{spec.name}.gemspec Rakefile .yardopts ],
-    %w[ CHANGELOG.md LICENSE.txt README.md ],
+    spec.extra_rdoc_files,
   ].flatten
 
-  spec.add_runtime_dependency 'attr_bool'            ,'~> 0.2'  # For attr_accessor?/attr_reader?
-  spec.add_runtime_dependency 'bimyou_segmenter'     ,'~> 1.2'  # For splitting Japanese sentences into words
-  spec.add_runtime_dependency 'cri'                  ,'~> 2.15' # For CLI commands/options
-  spec.add_runtime_dependency 'down'                 ,'~> 5.3'  # For downloading files (GetCmd)
-  spec.add_runtime_dependency 'highline'             ,'~> 2.0'  # For CLI input/output
-  spec.add_runtime_dependency 'http-cookie'          ,'~> 1.0'  # For parsing/setting cookies (BingScraper/Scraper)
-  spec.add_runtime_dependency 'japanese_deinflector' ,'~> 0.0'  # For unconjugating Japanese words (plain/dictionary form)
-  spec.add_runtime_dependency 'nokogiri'             ,'~> 1.13' # For scraping/hacking
-  spec.add_runtime_dependency 'psychgus'             ,'~> 1.3'  # For styling Psych YAML
-  spec.add_runtime_dependency 'public_suffix'        ,'~> 4.0'  # For parsing URL domain names
-  spec.add_runtime_dependency 'rainbow'              ,'~> 3.1'  # For CLI color output
-  spec.add_runtime_dependency 'rss'                  ,'~> 0.2'  # For scraping (BingScraper/Scraper)
-  spec.add_runtime_dependency 'rubyzip'              ,'~> 2.3'  # For extracting Zip files (GetCmd)
-  spec.add_runtime_dependency 'tiny_segmenter'       ,'~> 0.0'  # For splitting Japanese sentences into words
-  spec.add_runtime_dependency 'tty-progressbar'      ,'~> 0.18' # For CLI progress bars
-  spec.add_runtime_dependency 'tty-spinner'          ,'~> 0.9'  # For CLI spinning progress
+  run_dep = spec.method(:add_runtime_dependency)
+  run_dep[ 'attr_bool'           ,'~> 0.2'  ] # attr_accessor?/attr_reader?.
+  run_dep[ 'bimyou_segmenter'    ,'~> 1.2'  ] # Splitting Japanese sentences into words.
+  run_dep[ 'cri'                 ,'~> 2.15' ] # CLI commands/options.
+  run_dep[ 'down'                ,'~> 5.3'  ] # Downloading files (GetCmd).
+  run_dep[ 'highline'            ,'~> 2.0'  ] # CLI input/output.
+  run_dep[ 'http-cookie'         ,'~> 1.0'  ] # Parsing/Setting cookies [(Bing)Scraper].
+  run_dep[ 'japanese_deinflector','~> 0.0'  ] # Unconjugating Japanese words (dictionary form).
+  run_dep[ 'nokogiri'            ,'~> 1.13' ] # Scraping/Hacking.
+  run_dep[ 'psychgus'            ,'~> 1.3'  ] # Styling Psych YAML.
+  run_dep[ 'public_suffix'       ,'~> 4.0'  ] # Parsing URL domain names.
+  run_dep[ 'rainbow'             ,'~> 3.1'  ] # CLI color output.
+  run_dep[ 'rss'                 ,'~> 0.2'  ] # Scraping [(Bing)Scraper].
+  run_dep[ 'rubyzip'             ,'~> 2.3'  ] # Extracting Zip files (GetCmd).
+  run_dep[ 'tiny_segmenter'      ,'~> 0.0'  ] # Splitting Japanese sentences into words.
+  run_dep[ 'tty-progressbar'     ,'~> 0.18' ] # CLI progress bars.
+  run_dep[ 'tty-spinner'         ,'~> 0.9'  ] # CLI spinning progress.
 
-  spec.add_development_dependency 'bundler'   ,'~> 2.3'
-  spec.add_development_dependency 'minitest'  ,'~> 5.15'
-  spec.add_development_dependency 'rake'      ,'~> 13.0'
-  spec.add_development_dependency 'raketeer'  ,'~> 0.2'  # For extra Rake tasks
-  spec.add_development_dependency 'rdoc'      ,'~> 6.4'  # For YARDoc RDoc (*.rb)
-  spec.add_development_dependency 'redcarpet' ,'~> 3.5'  # For YARDoc Markdown (*.md)
-  spec.add_development_dependency 'yard'      ,'~> 0.9'  # For documentation
-  spec.add_development_dependency 'yard_ghurt','~> 1.2'  # For extra YARDoc Rake tasks
+  dev_dep = spec.method(:add_development_dependency)
+  dev_dep[ 'bundler'   ,'~> 2.3'  ]
+  dev_dep[ 'minitest'  ,'~> 5.16' ]
+  dev_dep[ 'rake'      ,'~> 13.0' ]
+  dev_dep[ 'raketeer'  ,'~> 0.2'  ] # Extra Rake tasks.
+  dev_dep[ 'rdoc'      ,'~> 6.4'  ] # YARDoc RDoc (*.rb).
+  dev_dep[ 'redcarpet' ,'~> 3.5'  ] # YARDoc Markdown (*.md).
+  dev_dep[ 'yard'      ,'~> 0.9'  ] # Doc.
+  dev_dep[ 'yard_ghurt','~> 1.2'  ] # Extra YARDoc Rake tasks.
 
   spec.post_install_message = <<~MSG
     +=============================================================================+
@@ -81,13 +90,7 @@ Gem::Specification.new do |spec|
     | Changelog: #{spec.metadata['changelog_uri']}
     +=============================================================================+
   MSG
-  #puts spec.post_install_message.split("\n").map(&:length).max
-
-  spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
 
-  spec.rdoc_options = [
-    '--hyperlink-all','--show-hash',
-    '--title',"NHKore v#{NHKore::VERSION} Doc",
-    '--main','README.md',
-  ]
+  # Uncomment to see max line length:
+  #puts spec.post_install_message.split("\n").map(&:length).max
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nhkore
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.14
 platform: ruby
 authors:
 - Jonathan Bradley Whited
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-27 00:00:00.000000000 Z
+date: 2022-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: attr_bool
@@ -254,14 +254,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.15'
+        version: '5.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.15'
+        version: '5.16'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -412,7 +412,7 @@ metadata:
   changelog_uri: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md
 post_install_message: |
   +=============================================================================+
-  | NHKore v0.3.13
+  | NHKore v0.3.14
   |
   | You can now use [nhkore] on the command line.
   |
@@ -425,7 +425,7 @@ rdoc_options:
 - "--hyperlink-all"
 - "--show-hash"
 - "--title"
-- NHKore v0.3.13 Doc
+- NHKore v0.3.14 Doc
 - "--main"
 - README.md
 require_paths: