nhkore 0.3.19 → 0.3.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a63aff9e86de6a678a4b4ad3defa9bf7d28577dabad0ef0a69747ddce1224219
-  data.tar.gz: 12668a0e95694198c928644d51be450c485c0947f5741dd01ab2ff8e20b0e760
+  metadata.gz: 6d6942febe79f05d1cbd53e40f9048671ee0462d16e22945a6b7bdb3bb5bb2ae
+  data.tar.gz: 226acf4e93e6b95b475a1d42bc7a0d2dff82d5647613164fc2feba882e24b6c7
 SHA512:
-  metadata.gz: 4c7981a84189176febf8156251b850e604ef1c447e1c78d0a8af5a4a509b68c0cf564b54e7cfac4582d1c572a50da0f05f9e268b19e4ac9d170ae0806688f68d
-  data.tar.gz: fd540fd9952b5377fcf4db72e4a5e71724d0a8d20ae256ecc4bb8293c672cd0fea1284e2639444fe8fbd892859fbac85ebc66fbabe96c51fdd60322e96f8419a
+  metadata.gz: 40d4b5513b9a3d22e4969ef7d88c477d4c1dc118165660e751c8c9d742305d74470494597e5e12f61e15af9dc4c3f4e0bccdd82ac7e8488cb2a312a9749ada20
+  data.tar.gz: 00307ecff6363a7f0b80595fc7ec14a01f3e9e303d5d3023e084883a075bed12464a388ca16b6ff6ca143e993bdb5834493d361084452dd61846567173f0ef7f

data/CHANGELOG.md

@@ -1,12 +1,18 @@
 # Changelog | NHKore
 
-All notable changes to this project will be documented in this file.
+- [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0)
+- [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html)
 
-Format is based on [Keep a Changelog v1.0.0](https://keepachangelog.com/en/1.0.0),
-and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+- https://github.com/esotericpig/nhkore/compare/v0.3.22...v0.3
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.19...HEAD)
--
+
+## [v0.3.22] - 2025-04-30
+
+### Changed
+- Put v0.3 in its own branch to prepare for v0.4, which will heavily change.
+- Changed v0.3 links to use v0.3 branch.
+- Reverted the removing of `UserAgents` for v0.3 only.
 
 
 ## [v0.3.19] - 2025-04-28

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nhkore (0.3.19)
+    nhkore (0.3.22)
       attr_bool (~> 0.2)
       bimyou_segmenter (~> 1.2)
       cri (~> 2.15)
@@ -14,7 +14,6 @@ PATH
       psychgus (~> 1.3)
       public_suffix (~> 6.0)
       rainbow (~> 3.1)
-      ronin-web-user_agents (~> 0.1)
       rss (~> 0.3)
       rubyzip (~> 2.4)
       tiny_segmenter (~> 0.0)
@@ -50,7 +49,7 @@ GEM
       stringio
     psychgus (1.3.5)
       psych (>= 3.0)
-    public_suffix (6.0.1)
+    public_suffix (6.0.2)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.2.1)
@@ -62,7 +61,6 @@ GEM
     reline (0.6.1)
       io-console (~> 0.5)
     rexml (3.4.1)
-    ronin-web-user_agents (0.1.1)
     rss (0.3.1)
       rexml
     rubyzip (2.4.1)

data/README.md

@@ -732,7 +732,7 @@ if !File.exist?(file)
 end
 ```
 
-### Util & DatetimeParser
+### Util, UserAgents, & DatetimeParser
 
 These provide a variety of useful methods/constants.
 
@@ -740,6 +740,7 @@ Here are some of the most useful ones:
 
 ```Ruby
 require 'nhkore/datetime_parser'
+require 'nhkore/user_agents'
 require 'nhkore/util'
 
 include NHKore
@@ -747,6 +748,10 @@ include NHKore
 puts '======='
 puts '[ Net ]'
 puts '======='
+# Get a random User Agent for HTTP header field 'User-Agent'.
+# - This is used by default in Scraper/SearchScraper.
+puts "User-Agent:  #{UserAgents.sample()}"
+
 uri = URI('https://www.bing.com/search?q=nhk')
 Util.replace_uri_query!(uri,q: 'banana')
 

data/lib/nhkore/cli/get_cmd.rb

@@ -9,6 +9,7 @@
 #++
 
 require 'nhkore/util'
+require 'nhkore/version'
 
 module NHKore
 module CLI
@@ -16,7 +17,8 @@ module CLI
     DEFAULT_GET_CHUNK_SIZE = 4 * 1024
     DEFAULT_GET_URL_LENGTH = 11_000_000 # Just a generous estimation used as a fallback; may be outdated.
     GET_URL_FILENAME = 'nhkore-core.zip'
-    GET_URL = "https://github.com/esotericpig/nhkore/releases/latest/download/#{GET_URL_FILENAME}".freeze
+    GET_URL = "https://github.com/esotericpig/nhkore/releases/download/v#{NHKore::VERSION}" \
+              "/#{GET_URL_FILENAME}".freeze
 
     def build_get_cmd
       app = self
@@ -36,9 +38,7 @@ module CLI
         DESC
 
         option :o,:out,'directory to save downloaded files to',argument: :required,default: Util::CORE_DIR,
-            transform: lambda { |value|
-              app.check_empty_opt(:out,value)
-            }
+               transform: ->(value) { app.check_empty_opt(:out,value) }
         flag nil,:'show-url','show download URL and exit (for downloading manually)' do |_value,_cmd|
           puts GET_URL
           exit
@@ -69,7 +69,7 @@ module CLI
       out_dir = @cmd_opts[:out]
 
       begin
-        start_spin('Opening URL')
+        start_spin("Opening URL: #{GET_URL} ")
 
         begin
           down = Down::NetHttp.open(GET_URL,rewindable: false,**@scraper_kargs)

data/lib/nhkore/lib.rb

@@ -31,6 +31,7 @@ require 'nhkore/search_link'
 require 'nhkore/search_scraper'
 require 'nhkore/sifter'
 require 'nhkore/splitter'
+require 'nhkore/user_agents'
 require 'nhkore/util'
 require 'nhkore/variator'
 require 'nhkore/version'

data/lib/nhkore/scraper.rb

@@ -11,9 +11,9 @@
 require 'attr_bool'
 require 'nokogiri'
 require 'open-uri'
-require 'ronin/web/user_agents'
 
 require 'nhkore/error'
+require 'nhkore/user_agents'
 require 'nhkore/util'
 
 module NHKore
@@ -23,7 +23,7 @@ module NHKore
     DEFAULT_HEADER = {
       # See for better ones:
       # - https://www.useragentstring.com/pages/Chrome/
-      'user-agent' => Ronin::Web::UserAgents.random,
+      'user-agent' => UserAgents.sample,
 
       'accept' => 'text/html,application/xhtml+xml,application/xml,application/rss+xml,text/xml;' \
                   'q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',

data/lib/nhkore/search_scraper.rb

@@ -111,14 +111,14 @@ module NHKore
       super(url,**kargs)
     end
 
-    # FIXME: Bing no longer allows `count`.
-    # rubocop:disable Lint/UnusedMethodArgument
     def self.build_url(site,count: DEFAULT_RESULT_COUNT,**_kargs)
       url = ''.dup
 
       url << 'https://www.bing.com/search?'
       url << URI.encode_www_form(
         q: "site:#{site}",
+        count: count,
+
         qs: 'n',
         sp: '-1',
         lq: '0',
@@ -131,15 +131,13 @@ module NHKore
 
       return url
     end
-    # rubocop:enable Lint/UnusedMethodArgument
 
     def scrape(slinks,page = NextPage.new())
-      next_page,_link_count = scrape_html(slinks,page)
+      next_page,link_count = scrape_html(slinks,page)
 
-      # FIXME: Bing no longer allows RSS pages after the first page.
-      # if link_count <= 0
-      #   scrape_rss(slinks,page,next_page)
-      # end
+      if link_count <= 0
+        scrape_rss(slinks,page,next_page)
+      end
 
       return next_page
     end

data/lib/nhkore/sifter.rb

@@ -322,7 +322,7 @@ module NHKore
     end
 
     def sift
-      master_article = Article.new
+      result = Article.new
 
       @articles.each do |article|
         next if filter?(article)
@@ -333,11 +333,11 @@ module NHKore
           next if word.freq <= 1
           next if word.word =~ /\p{Latin}|[[:digit:]]/
 
-          master_article.add_word(word,use_freq: true)
+          result.add_word(word,use_freq: true)
         end
       end
 
-      words = master_article.words.values
+      words = result.words.values
 
       words.sort! do |word1,word2|
         # Order by freq DESC (most frequent words to top).