nhkore 0.3.4 → 0.3.9

data/lib/nhkore/splitter.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -26,24 +14,24 @@ require 'nhkore/util'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class Splitter
     def begin_split(str)
       return str
     end
-    
+
     def split(str)
       str = begin_split(str)
       str = end_split(str)
-      
+
       return str
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BasicSplitter < Splitter
@@ -51,43 +39,43 @@ module NHKore
       return str.split(Util::NORMALIZE_STR_REGEX)
     end
   end
-  
+
   ###
   # @since  0.2.0
   ###
   class BimyouSplitter < Splitter
     def initialize(*)
       require 'bimyou_segmenter'
-      
+
       super
     end
-    
+
     def end_split(str)
       return BimyouSegmenter.segment(str,symbol: false,white_space: false)
     end
   end
-  
+
   ###
   # @since  0.2.0
   ###
   class TinySplitter < Splitter
     attr_accessor :tiny
-    
+
     def initialize(*)
       require 'tiny_segmenter'
-      
+
       super
-      
-      @tiny = TinySegmenter.new()
+
+      @tiny = TinySegmenter.new
     end
-    
+
     def end_split(str)
       return @tiny.segment(str,ignore_punctuation: true)
     end
   end
-  
+
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   class BestSplitter < BimyouSplitter

data/lib/nhkore/user_agents.rb

@@ -1,39 +1,33 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
-if $0 == __FILE__
-  # `gem install 'user-agent-randomizer'` required.
-  
+if $PROGRAM_NAME == __FILE__
+  require 'bundler/inline'
+
+  gemfile do
+    source 'https://rubygems.org'
+
+    gem 'user-agent-randomizer','~> 0.2',require: false
+  end
+
   require 'set'
   require 'user_agent_randomizer'
-  
-  agents = Set.new()
-  
+
+  agents = Set.new
+
   while agents.length < 1111
     agents.add(UserAgentRandomizer::UserAgent.fetch(type: 'desktop_browser').string)
   end
-  
-  agents.each() do |agent|
+
+  agents.each do |agent|
     puts "'#{agent}',"
   end
 end
@@ -41,26 +35,27 @@ end
 module NHKore
   ###
   # 1111 user-agents produced using the +user-agent-randomizer+ gem.
-  # 
+  #
   # The gem is really old and had a lot of warnings, so decided to make this class.
   # Maybe I'll fork the gem and maintain a new version in the future...
-  # 
-  # @author Jonathan Bradley Whited (@esotericpig)
+  #
+  # @author Jonathan Bradley Whited
   # @since  0.2.1
   ###
   class UserAgents
     attr_accessor :data
-    
-    def self.sample()
-      return UserAgents.new().data.sample()
+
+    def self.sample
+      return UserAgents.new.data.sample
     end
-    
+
     # Decided to store the data in an instance variable (instead of a constant)
     # because we don't need all of the data in memory after getting just 1
     # sample, even though it's slower.
-    def initialize()
+    def initialize
       super()
-      
+
+      # rubocop:disable all
       @data = [
 'ELinks/0.11.3 (textmode; Darwin 10.2.0 i386; 80x24-2)',
 'Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; de) Opera 10.10',
@@ -1174,6 +1169,7 @@ module NHKore
 'Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.1a) Gecko/20020611',
 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0',
       ]
+      # rubocop:enable all
     end
   end
 end

data/lib/nhkore/util.rb

@@ -1,23 +1,11 @@
-#!/usr/bin/env ruby
 # encoding: UTF-8
 # frozen_string_literal: true
 
 #--
 # This file is part of NHKore.
-# Copyright (c) 2020 Jonathan Bradley Whited (@esotericpig)
-# 
-# NHKore is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# 
-# NHKore is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public License
-# along with NHKore.  If not, see <https://www.gnu.org/licenses/>.
+# Copyright (c) 2020-2021 Jonathan Bradley Whited
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
 #++
 
 
@@ -29,110 +17,110 @@ require 'uri'
 
 module NHKore
   ###
-  # @author Jonathan Bradley Whited (@esotericpig)
+  # @author Jonathan Bradley Whited
   # @since  0.2.0
   ###
   module Util
     CORE_DIR = 'core'
     WEB_DIR = 'web'
-    
+
     JST_OFFSET = '+09:00' # Japan Standard Time (JST) time zone offset from UTC
     JST_OFFSET_HOUR = 9
     JST_OFFSET_MIN = 0
-    
-    HIRAGANA_REGEX = /\p{Hiragana}/
+
+    HIRAGANA_REGEX = /\p{Hiragana}/.freeze
     JPN_SPACE = "\u3000" # Must be double-quoted for escape chars
-    KANA_REGEX = /\p{Hiragana}|\p{Katakana}/
-    KANJI_REGEX = /\p{Han}/ # Han probably stands for Hanzi?
-    KATAKANA_REGEX = /\p{Katakana}/
-    NORMALIZE_STR_REGEX = /[^[[:alpha:]]]+/
-    STRIP_WEB_STR_REGEX = /(\A[[:space:]]+)|([[:space:]]+\z)/
-    WEB_SPACES_REGEX = /[[:space:]]+/
-    
-    def self.jst_now()
-      return Time.now().getlocal(JST_OFFSET)
-    end
-    
-    JST_YEAR = jst_now().year
+    KANA_REGEX = /\p{Hiragana}|\p{Katakana}/.freeze
+    KANJI_REGEX = /\p{Han}/.freeze # Han probably stands for Hanzi?
+    KATAKANA_REGEX = /\p{Katakana}/.freeze
+    NORMALIZE_STR_REGEX = /[^[[:alpha:]]]+/.freeze
+    STRIP_WEB_STR_REGEX = /(\A[[:space:]]+)|([[:space:]]+\z)/.freeze
+    WEB_SPACES_REGEX = /[[:space:]]+/.freeze
+
+    def self.jst_now
+      return Time.now.getlocal(JST_OFFSET)
+    end
+
+    JST_YEAR = jst_now.year
     MAX_SANE_YEAR = JST_YEAR + 1 # +1 Justin Case for time zone differences at the end of the year
-    
+
     # NHK was founded in 1924/25.
     # - https://www.nhk.or.jp/bunken/english/about/history.html
     # - https://en.wikipedia.org/wiki/NHK
     # However, when was the website first created?
     MIN_SANE_YEAR = 1924
-    
+
     def self.dir_str?(str)
-      return str.match?(/[\/\\]\s*\z/)
+      return str.match?(%r{[/\\]\s*\z/})
     end
-    
+
     def self.domain(host,clean: true)
       require 'public_suffix'
-      
+
       domain = PublicSuffix.domain(host)
-      domain = unspace_web_str(domain).downcase() if !domain.nil?() && clean
-      
+      domain = unspace_web_str(domain).downcase if !domain.nil? && clean
+
       return domain
     end
-    
+
     def self.dump_yaml(obj,flow_level: 8,stylers: nil)
       require 'psychgus'
-      
+
       stylers = Array(stylers)
-      
+
       return Psychgus.dump(obj,
         deref_aliases: true, # Dereference aliases for load_yaml()
         header: true, # %YAML [version]
-        line_width: 10000, # Try not to wrap; ichiman!
+        line_width: 10_000, # Try not to wrap; ichiman!
         stylers: [
           Psychgus::FlowStyler.new(flow_level), # Put extra details on one line (flow/inline style)
           Psychgus::NoSymStyler.new(cap: false), # Remove symbols, don't capitalize
-          Psychgus::NoTagStyler.new(), # Remove class names (tags)
+          Psychgus::NoTagStyler.new, # Remove class names (tags)
         ].concat(stylers),
       )
     end
-    
+
     def self.empty_web_str?(str)
-      return str.nil?() || strip_web_str(str).empty?()
+      return str.nil? || strip_web_str(str).empty?
     end
-    
+
     def self.escape_html(str)
       str = CGI.escapeHTML(str)
       str = str.gsub("\n",'<br>')
-      
+
       return str
     end
-    
+
     def self.filename_str?(str)
       # Do not use "!dir_str?()"! It's not the same meaning!
-      return !str.match?(/[\/\\]/)
+      return !str.match?(%r{[/\\]})
     end
-    
+
     def self.hiragana?(str)
       return HIRAGANA_REGEX =~ str
     end
-    
+
     # This doesn't modify the hour/minute according to {JST_OFFSET},
     # but instead, it just drops {JST_OFFSET} into it without adjusting it.
     def self.jst_time(time)
       return Time.new(time.year,time.month,time.day,time.hour,time.min,time.sec,JST_OFFSET)
     end
-    
+
     def self.kana?(str)
       return KANA_REGEX =~ str
     end
-    
+
     def self.kanji?(str)
       return KANJI_REGEX =~ str
     end
-    
+
     def self.katakana?(str)
       return KATAKANA_REGEX =~ str
     end
-    
+
     def self.load_yaml(data,file: nil,**kargs)
       require 'psychgus'
-      
+
       return Psych.safe_load(data,
         aliases: false,
         filename: file,
@@ -142,72 +130,72 @@ module NHKore
         **kargs,
       )
     end
-    
+
     def self.normalize_str(str)
       return str.gsub(NORMALIZE_STR_REGEX,'')
     end
-    
+
     def self.reduce_jpn_space(str)
       # Do not strip; use a Japanese space
       return str.gsub(WEB_SPACES_REGEX,JPN_SPACE)
     end
-    
+
     def self.reduce_space(str)
       return str.gsub(WEB_SPACES_REGEX,' ')
     end
-    
+
     def self.replace_uri_query!(uri,**new_query)
-      return uri if new_query.empty?()
-      
+      return uri if new_query.empty?
+
       query = uri.query
-      query = query.nil?() ? [] : URI.decode_www_form(query)
-      
+      query = query.nil? ? [] : URI.decode_www_form(query)
+
       # First, remove the old ones.
-      if !query.empty?()
-        new_query_keys = Set.new(new_query.keys.map() {|key|
-          unspace_web_str(key.to_s()).downcase()
-        })
-        
-        query.filter!() do |q|
-          if q.nil?() || q.empty?()
+      if !query.empty?
+        new_query_keys = Set.new(new_query.keys.map do |key|
+          unspace_web_str(key.to_s).downcase
+        end)
+
+        query.filter! do |q|
+          if q.nil? || q.empty?
             false
           else
-            key = unspace_web_str(q[0].to_s()).downcase()
-            
+            key = unspace_web_str(q[0].to_s).downcase
+
             !new_query_keys.include?(key)
           end
         end
       end
-      
+
       # Next, add the new ones.
-      new_query.each() do |key,value|
-        query << [key,value.nil?() ? '' : value]
+      new_query.each do |key,value|
+        query << [key,value.nil? ? '' : value]
       end
-      
+
       uri.query = URI.encode_www_form(query)
-      
+
       return uri
     end
-    
+
     def self.sane_year?(year)
       return year >= MIN_SANE_YEAR && year <= MAX_SANE_YEAR
     end
-    
+
     # String's normal strip() method doesn't work with special Unicode/HTML white space.
     def self.strip_web_str(str)
       # After testing with Benchmark, this is slower than one regex.
       #str = str.gsub(/\A[[:space:]]+/,'')
       #str = str.gsub(/[[:space:]]+\z/,'')
-      
+
       str = str.gsub(STRIP_WEB_STR_REGEX,'')
-      
+
       return str
     end
-    
+
     def self.unspace_web_str(str)
       return str.gsub(WEB_SPACES_REGEX,'')
     end
-    
+
     def self.warn(msg,uplevel: 1)
       Kernel.warn(msg,uplevel: uplevel)
     end