nhkore 0.2.0 → 0.3.0

data/lib/nhkore/util.rb

@@ -24,7 +24,9 @@
 require 'cgi'
 require 'psychgus'
 require 'public_suffix'
+require 'set'
 require 'time'
+require 'uri'
 
 
 module NHKore
@@ -69,7 +71,7 @@ module NHKore
     
     def self.domain(host,clean: true)
       domain = PublicSuffix.domain(host)
-      domain = unspace_web_str(domain).downcase() if clean
+      domain = unspace_web_str(domain).downcase() if !domain.nil?() && clean
       
       return domain
     end
@@ -164,6 +166,39 @@ module NHKore
       return str.gsub(WEB_SPACES_REGEX,' ')
     end
     
+    def self.replace_uri_query!(uri,**new_query)
+      return uri if new_query.empty?()
+      
+      query = uri.query
+      query = query.nil?() ? [] : URI.decode_www_form(query)
+      
+      # First, remove the old ones.
+      if !query.empty?()
+        new_query_keys = Set.new(new_query.keys.map() {|key|
+          unspace_web_str(key.to_s()).downcase()
+        })
+        
+        query.filter!() do |q|
+          if q.nil?() || q.empty?()
+            false
+          else
+            key = unspace_web_str(q[0].to_s()).downcase()
+            
+            !new_query_keys.include?(key)
+          end
+        end
+      end
+      
+      # Next, add the new ones.
+      new_query.each() do |key,value|
+        query << [key,value.nil?() ? '' : value]
+      end
+      
+      uri.query = URI.encode_www_form(query)
+      
+      return uri
+    end
+    
     def self.sane_year?(year)
       return year >= MIN_SANE_YEAR && year <= MAX_SANE_YEAR
     end

data/lib/nhkore/version.rb

@@ -22,5 +22,5 @@
 
 
 module NHKore
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/nhkore.gemspec

@@ -34,10 +34,9 @@ Gem::Specification.new() do |spec|
   spec.licenses    = ['LGPL-3.0-or-later']
   spec.homepage    = 'https://github.com/esotericpig/nhkore'
   spec.summary     = 'NHK News Web (Easy) word frequency (core) scraper for Japanese language learners.'
-  spec.description = <<-EOD.gsub(/\s{2,}/,' ').strip()
-    Scrapes NHK News Web (Easy) for the word frequency (core list) for Japanese language learners.
-    Includes a CLI app and a scraper library.
-  EOD
+  spec.description =
+    'Scrapes NHK News Web (Easy) for the word frequency (core list) for Japanese language learners.' \
+    ' Includes a CLI app and a scraper library.'
   
   spec.metadata = {
     'bug_tracker_uri' => 'https://github.com/esotericpig/nhkore/issues',
@@ -60,19 +59,20 @@ Gem::Specification.new() do |spec|
   
   spec.requirements << 'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html'
   
-  spec.add_runtime_dependency 'bimyou_segmenter'    ,'~> 1.2'  # For splitting Japanese sentences into words
-  spec.add_runtime_dependency 'cri'                 ,'~> 2.15' # For CLI commands/options
-  spec.add_runtime_dependency 'down'                ,'~> 5.1'  # For downloading files (GetCmd)
-  spec.add_runtime_dependency 'highline'            ,'~> 2.0'  # For CLI input/output
-  spec.add_runtime_dependency 'japanese_deinflector','~> 0.0'  # For unconjugating Japanese words (plain/dictionary form)
-  spec.add_runtime_dependency 'nokogiri'            ,'~> 1.10' # For scraping/hacking
-  spec.add_runtime_dependency 'psychgus'            ,'~> 1.2'  # For styling Psych YAML
-  spec.add_runtime_dependency 'public_suffix'       ,'~> 4.0'  # For parsing URL domain names
-  spec.add_runtime_dependency 'rainbow'             ,'~> 3.0'  # For CLI color output
-  spec.add_runtime_dependency 'rubyzip'             ,'~> 2.3'  # For extracting Zip files (GetCmd)
-  spec.add_runtime_dependency 'tiny_segmenter'      ,'~> 0.0'  # For splitting Japanese sentences into words
-  spec.add_runtime_dependency 'tty-progressbar'     ,'~> 0.17' # For CLI progress bars
-  spec.add_runtime_dependency 'tty-spinner'         ,'~> 0.9'  # For CLI spinning progress
+  spec.add_runtime_dependency 'bimyou_segmenter'     ,'~> 1.2'  # For splitting Japanese sentences into words
+  spec.add_runtime_dependency 'cri'                  ,'~> 2.15' # For CLI commands/options
+  spec.add_runtime_dependency 'down'                 ,'~> 5.1'  # For downloading files (GetCmd)
+  spec.add_runtime_dependency 'highline'             ,'~> 2.0'  # For CLI input/output
+  spec.add_runtime_dependency 'http-cookie'          ,'~> 1.0'  # For parsing/setting cookies (BingScraper/Scraper)
+  spec.add_runtime_dependency 'japanese_deinflector' ,'~> 0.0'  # For unconjugating Japanese words (plain/dictionary form)
+  spec.add_runtime_dependency 'nokogiri'             ,'~> 1.10' # For scraping/hacking
+  spec.add_runtime_dependency 'psychgus'             ,'~> 1.2'  # For styling Psych YAML
+  spec.add_runtime_dependency 'public_suffix'        ,'~> 4.0'  # For parsing URL domain names
+  spec.add_runtime_dependency 'rainbow'              ,'~> 3.0'  # For CLI color output
+  spec.add_runtime_dependency 'rubyzip'              ,'~> 2.3'  # For extracting Zip files (GetCmd)
+  spec.add_runtime_dependency 'tiny_segmenter'       ,'~> 0.0'  # For splitting Japanese sentences into words
+  spec.add_runtime_dependency 'tty-progressbar'      ,'~> 0.17' # For CLI progress bars
+  spec.add_runtime_dependency 'tty-spinner'          ,'~> 0.9'  # For CLI spinning progress
   
   spec.add_development_dependency 'bundler'   ,'~> 2.1'
   spec.add_development_dependency 'minitest'  ,'~> 5.14'
@@ -83,5 +83,17 @@ Gem::Specification.new() do |spec|
   spec.add_development_dependency 'yard'      ,'~> 0.9'  # For documentation
   spec.add_development_dependency 'yard_ghurt','~> 1.2'  # For extra YARDoc Rake tasks
   
-  spec.post_install_message = "You can now use [#{spec.executables.join(', ')}] on the command line."
+  spec.post_install_message = <<-EOM
+  
+  NHKore v#{NHKore::VERSION}
+  
+  You can now use [#{spec.executables.join(', ')}] on the command line.
+  
+  Homepage:  #{spec.homepage}
+  
+  Code:      #{spec.metadata['source_code_uri']}
+  Changelog: #{spec.metadata['changelog_uri']}
+  Bugs:      #{spec.metadata['bug_tracker_uri']}
+  
+  EOM
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nhkore
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jonathan Bradley Whited (@esotericpig)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-31 00:00:00.000000000 Z
+date: 2020-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bimyou_segmenter
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: http-cookie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: japanese_deinflector
   requirement: !ruby/object:Gem::Requirement
@@ -324,10 +338,10 @@ files:
 - lib/nhkore/article.rb
 - lib/nhkore/article_scraper.rb
 - lib/nhkore/cleaner.rb
-- lib/nhkore/cli/bing_cmd.rb
 - lib/nhkore/cli/fx_cmd.rb
 - lib/nhkore/cli/get_cmd.rb
 - lib/nhkore/cli/news_cmd.rb
+- lib/nhkore/cli/search_cmd.rb
 - lib/nhkore/cli/sift_cmd.rb
 - lib/nhkore/defn.rb
 - lib/nhkore/dict.rb
@@ -343,6 +357,7 @@ files:
 - lib/nhkore/search_scraper.rb
 - lib/nhkore/sifter.rb
 - lib/nhkore/splitter.rb
+- lib/nhkore/user_agents.rb
 - lib/nhkore/util.rb
 - lib/nhkore/variator.rb
 - lib/nhkore/version.rb
@@ -359,7 +374,10 @@ metadata:
   changelog_uri: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md
   homepage_uri: https://github.com/esotericpig/nhkore
   source_code_uri: https://github.com/esotericpig/nhkore
-post_install_message: You can now use [nhkore] on the command line.
+post_install_message: "  \n  NHKore v0.3.0\n  \n  You can now use [nhkore] on the
+  command line.\n  \n  Homepage:  https://github.com/esotericpig/nhkore\n  \n  Code:
+  \     https://github.com/esotericpig/nhkore\n  Changelog: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md\n
+  \ Bugs:      https://github.com/esotericpig/nhkore/issues\n  \n"
 rdoc_options: []
 require_paths:
 - lib