nhkore 0.3.1 → 0.3.6

data/Rakefile

@@ -35,7 +35,7 @@ require 'nhkore/version'
 
 PKG_DIR = 'pkg'
 
-CLEAN.exclude('.git/','stock/')
+CLEAN.exclude('{.git,core,stock}/**/*')
 CLOBBER.include('doc/',File.join(PKG_DIR,''))
 
 
@@ -49,7 +49,7 @@ desc "Package '#{File.join(NHKore::Util::CORE_DIR,'')}' data as a Zip file into
 task :pkg_core do |task|
   mkdir_p PKG_DIR
   
-  pattern = File.join(NHKore::Util::CORE_DIR,'*.{csv,html,yml}')
+  pattern = File.join(NHKore::Util::CORE_DIR,'*.{csv,html,json,yml}')
   zip_file = File.join(PKG_DIR,'nhkore-core.zip')
   
   sh 'zip','-9rv',zip_file,*Dir.glob(pattern).sort()
@@ -59,17 +59,57 @@ Rake::TestTask.new() do |task|
   task.libs = ['lib','test']
   task.pattern = File.join('test','**','*_test.rb')
   task.description += ": '#{task.pattern}'"
-  task.verbose = true
+  task.verbose = false
   task.warning = true
 end
 
-YARD::Rake::YardocTask.new() do |task|
-  task.files = [File.join('lib','**','*.rb')]
+# If you need to run a part after the 1st part,
+# just type 'n' to not overwrite the file and then 'y' for continue.
+desc "Update '#{File.join(NHKore::Util::CORE_DIR,'')}' files for release"
+task :update_core do |task|
+  require 'highline'
+  
+  CONTINUE_MSG = "\nContinue (y/n)? "
+  
+  cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
+  hl = HighLine.new()
+  
+  next unless sh(*cmd,'se','ez','bing')
+  next unless hl.agree(CONTINUE_MSG)
+  puts
   
-  task.options += ['--files','CHANGELOG.md,LICENSE.txt']
-  task.options += ['--readme','README.md']
+  next unless sh(*cmd,'news','-s','100','ez')
+  next unless hl.agree(CONTINUE_MSG)
+  puts
+  
+  next unless sh(*cmd,'sift','-e','csv' ,'ez')
+  next unless sh(*cmd,'sift','-e','html','ez')
+  next unless sh(*cmd,'sift','-e','json','ez')
+  next unless sh(*cmd,'sift','-e','yml' ,'ez')
+end
+
+# @since 0.3.6
+desc 'Update showcase file for release'
+task :update_showcase do |task|
+  require 'highline'
   
-  task.options << '--protected' # Show protected methods
+  SHOWCASE_FILE = File.join('.','nhkore-ez.html')
+  
+  hl = HighLine.new()
+  
+  next unless sh('ruby','-w','./lib/nhkore.rb',
+    'sift','ez','--no-eng',
+    '--out',SHOWCASE_FILE,
+  )
+  
+  next unless hl.agree("\nMove the file (y/n)? ")
+  puts
+  next unless sh('mv','-iv',SHOWCASE_FILE,
+    File.join('..','esotericpig.github.io','showcase',''),
+  )
+end
+
+YARD::Rake::YardocTask.new() do |task|
   task.options += ['--template-path',File.join('yard','templates')]
   task.options += ['--title',"NHKore v#{NHKore::VERSION} Doc"]
 end

data/lib/nhkore.rb

@@ -29,28 +29,7 @@ if TESTING
 end
 
 require 'nhkore/app'
-require 'nhkore/article'
-require 'nhkore/article_scraper'
-require 'nhkore/cleaner'
-require 'nhkore/defn'
-require 'nhkore/dict'
-require 'nhkore/dict_scraper'
-require 'nhkore/entry'
-require 'nhkore/error'
-require 'nhkore/fileable'
-require 'nhkore/missingno'
-require 'nhkore/news'
-require 'nhkore/polisher'
-require 'nhkore/scraper'
-require 'nhkore/search_link'
-require 'nhkore/search_scraper'
-require 'nhkore/sifter'
-require 'nhkore/splitter'
-require 'nhkore/user_agents'
-require 'nhkore/util'
-require 'nhkore/variator'
-require 'nhkore/version'
-require 'nhkore/word'
+require 'nhkore/lib'
 
 require 'nhkore/cli/fx_cmd'
 require 'nhkore/cli/get_cmd'

data/lib/nhkore/app.rb

@@ -24,6 +24,7 @@
 require 'cri'
 require 'highline'
 require 'rainbow'
+require 'set'
 require 'tty-spinner'
 
 require 'nhkore/error'
@@ -151,7 +152,8 @@ module NHKore
       end
       
       if color.nil?()
-        color = ($stdout.tty?() && ENV['TERM'] != 'dumb')
+        # - https://no-color.org/
+        color = ($stdout.tty?() && ENV['TERM'] != 'dumb' && !ENV.key?('NO_COLOR'))
       end
       
       enable_color(color)

data/lib/nhkore/article.rb

@@ -33,11 +33,11 @@ module NHKore
   # @since  0.2.0
   ###
   class Article
-    attr_accessor :datetime
-    attr_accessor :futsuurl
+    attr_reader :datetime
+    attr_reader :futsuurl
     attr_accessor :sha256
     attr_accessor :title
-    attr_accessor :url
+    attr_reader :url
     attr_reader :words
     
     def initialize()
@@ -79,19 +79,18 @@ module NHKore
       
       coder[:datetime] = @datetime.nil?() ? @datetime : @datetime.iso8601()
       coder[:title] = @title
-      coder[:url] = @url
-      coder[:futsuurl] = @futsuurl
+      coder[:url] = @url.nil?() ? nil : @url.to_s()
+      coder[:futsuurl] = @futsuurl.nil?() ? nil : @futsuurl.to_s()
       coder[:sha256] = @sha256
       coder[:words] = @words
     end
     
     def self.load_data(key,hash)
-      datetime = hash[:datetime]
       words = hash[:words]
       
       article = Article.new()
       
-      article.datetime = Util.empty_web_str?(datetime) ? nil : Time.iso8601(datetime)
+      article.datetime = hash[:datetime]
       article.futsuurl = hash[:futsuurl]
       article.sha256 = hash[:sha256]
       article.title = hash[:title]
@@ -107,6 +106,24 @@ module NHKore
       return article
     end
     
+    def datetime=(value)
+      if value.is_a?(Time)
+        @datetime = value
+      else
+        @datetime = Util.empty_web_str?(value) ? nil : Time.iso8601(value)
+      end
+    end
+    
+    def futsuurl=(value)
+      # Don't store URI, store String.
+      @futsuurl = value.nil?() ? nil : value.to_s()
+    end
+    
+    def url=(value)
+      # Don't store URI, store String.
+      @url = value.nil?() ? nil : value.to_s()
+    end
+    
     def to_s(mini: false)
       s = ''.dup()
       

data/lib/nhkore/article_scraper.rb

@@ -21,6 +21,7 @@
 #++
 
 
+require 'attr_bool'
 require 'digest'
 
 require 'nhkore/article'
@@ -47,9 +48,9 @@ module NHKore
     attr_accessor :dict
     attr_reader :kargs
     attr_accessor :missingno
-    attr_accessor :mode
     attr_reader :polishers
     attr_accessor :splitter
+    attr_accessor? :strict
     attr_reader :variators
     attr_accessor :year
     
@@ -58,8 +59,8 @@ module NHKore
     #             [+nil+]     don't scrape/use it
     # @param missingno [Missingno] data to use as a fallback for Ruby words without kana/kanji,
     #                  instead of raising an error
-    # @param mode [nil,:lenient]
-    def initialize(url,cleaners: [BestCleaner.new()],datetime: nil,dict: :scrape,missingno: nil,mode: nil,polishers: [BestPolisher.new()],splitter: BestSplitter.new(),variators: [BestVariator.new()],year: nil,**kargs)
+    # @param strict [true,false]
+    def initialize(url,cleaners: [BestCleaner.new()],datetime: nil,dict: :scrape,missingno: nil,polishers: [BestPolisher.new()],splitter: BestSplitter.new(),strict: true,variators: [BestVariator.new()],year: nil,**kargs)
       super(url,**kargs)
       
       @cleaners = Array(cleaners)
@@ -67,9 +68,9 @@ module NHKore
       @dict = dict
       @kargs = kargs
       @missingno = missingno
-      @mode = mode
       @polishers = Array(polishers)
       @splitter = splitter
+      @strict = strict
       @variators = Array(variators)
       @year = year
     end
@@ -188,7 +189,7 @@ module NHKore
       tag = doc.css('div.article-body') if tag.length < 1
       
       # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
-      tag = doc.css('div#main') if tag.length < 1 && @mode == :lenient
+      tag = doc.css('div#main') if tag.length < 1 && !@strict
       
       if tag.length > 0
         text = Util.unspace_web_str(tag.text.to_s())
@@ -290,7 +291,7 @@ module NHKore
           
           retry
         else
-          raise e.exception("could not scrape dictionary at URL[#{dict_url}]: #{e}")
+          raise e.exception("could not scrape dictionary URL[#{dict_url}] at URL[#{@url}]: #{e}")
         end
       end
       
@@ -480,19 +481,24 @@ module NHKore
     
     def scrape_title(doc,article)
       tag = doc.css('h1.article-main__title')
+      tag_name = nil
       
-      if tag.length < 1 && @mode == :lenient
+      if tag.length < 1
+        # - https://www3.nhk.or.jp/news/easy/article/disaster_earthquake_illust.html
+        tag_name = 'h1.article-eq__title'
+        tag = doc.css(tag_name)
+      end
+      
+      if tag.length < 1 && !@strict
         # This shouldn't be used except for select sites.
         # - https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html
-        
         tag_name = 'div#main h2'
-        
-        Util.warn("using [#{tag_name}] for title at URL[#{@url}]")
-        
         tag = doc.css(tag_name)
       end
       
       if tag.length > 0
+        Util.warn("using [#{tag_name}] for title at URL[#{@url}]") unless tag_name.nil?()
+        
         result = scrape_and_add_words(tag,article)
         title = result.text
         
@@ -583,7 +589,7 @@ module NHKore
       end
       
       # As a last resort, use our user-defined fallbacks (if specified).
-      return @year unless Util.empty_web_str?(@year)
+      return @year.to_i() unless @year.nil?()
       return @datetime.year if !@datetime.nil?() && Util.sane_year?(@datetime.year)
       
       raise ScrapeError,"could not scrape year at URL[#{@url}]"
@@ -604,11 +610,10 @@ module NHKore
     end
     
     def warn_or_error(klass,msg)
-      case @mode
-      when :lenient
-        Util.warn(msg)
-      else
+      if @strict
         raise klass,msg
+      else
+        Util.warn(msg)
       end
     end
   end

data/lib/nhkore/cli/news_cmd.rb

@@ -23,6 +23,7 @@
 
 require 'time'
 
+require 'nhkore/datetime_parser'
 require 'nhkore/error'
 require 'nhkore/missingno'
 require 'nhkore/news'
@@ -57,7 +58,7 @@ module CLI
           date time to use as a fallback in cases when an article doesn't have one;
           format: YYYY-mm-dd H:M; example: 2020-03-30 15:30
         EOD
-          value = Time.strptime(value,'%Y-%m-%d %H:%M',&Util.method(:guess_year))
+          value = Time.strptime(value,'%Y-%m-%d %H:%M',&DatetimeParser.method(:guess_year))
           value = Util.jst_time(value)
           value
         end
@@ -237,7 +238,7 @@ module CLI
         dict: dict,
         is_file: is_file,
         missingno: missingno ? Missingno.new(news) : nil,
-        mode: lenient ? :lenient : nil,
+        strict: !lenient,
       })
       @news_dict_scraper_kargs = @scraper_kargs.merge({
         is_file: is_file,

data/lib/nhkore/cli/search_cmd.rb

@@ -83,7 +83,7 @@ module CLI
             key = key.to_s()
             
             if key.include?('show')
-              raise CLIError.new("must specify a sub command for option[#{key}]")
+              raise CLIError,"must specify a sub command for option[#{key}]"
             end
           end
           
@@ -283,7 +283,7 @@ module CLI
         puts "> Easy:    #{BingScraper.build_url(SearchScraper::YASASHII_SITE,count: count)}"
         puts "> Regular: #{BingScraper.build_url(SearchScraper::FUTSUU_SITE,count: count)}"
       else
-        raise CLIError.new('must specify a sub command for option[show-urls]')
+        raise CLIError,'must specify a sub command for option[show-urls]'
       end
       
       return true

data/lib/nhkore/cli/sift_cmd.rb

@@ -24,6 +24,7 @@
 require 'date'
 require 'time'
 
+require 'nhkore/datetime_parser'
 require 'nhkore/news'
 require 'nhkore/sifter'
 require 'nhkore/util'
@@ -39,27 +40,7 @@ module CLI
     DEFAULT_SIFT_EXT = :csv
     DEFAULT_SIFT_FUTSUU_FILE = "#{Sifter::DEFAULT_FUTSUU_FILE}{search.criteria}{file.ext}"
     DEFAULT_SIFT_YASASHII_FILE = "#{Sifter::DEFAULT_YASASHII_FILE}{search.criteria}{file.ext}"
-    SIFT_EXTS = [:csv,:htm,:html,:yaml,:yml]
-    
-    # Order matters.
-    SIFT_DATETIME_FMTS = [
-      '%Y-%m-%d %H:%M',
-      '%Y-%m-%d %H',
-      '%Y-%m-%d',
-      '%m-%d %H:%M',
-      '%Y-%m %H:%M',
-      '%m-%d %H',
-      '%Y-%m %H',
-      '%m-%d',
-      '%Y-%m',
-      '%d %H:%M',
-      '%y %H:%M',
-      '%d %H',
-      '%Y %H',
-      '%H:%M',
-      '%d',
-      '%Y'
-    ]
+    SIFT_EXTS = [:csv,:htm,:html,:json,:yaml,:yml]
     
     attr_accessor :sift_datetime_text
     attr_accessor :sift_search_criteria
@@ -90,7 +71,11 @@ module CLI
           '9' (9th of Current Year & Month)
         EOD
           app.sift_datetime_text = value # Save the original value for the file name
-          value = app.parse_sift_datetime(value)
+          
+          value = DatetimeParser.parse_range(value)
+          
+          app.check_empty_opt(:datetime,value) if value.nil?()
+          
           value
         end
         option :e,:ext,<<-EOD,argument: :required,default: DEFAULT_SIFT_EXT,transform: -> (value) do
@@ -211,96 +196,6 @@ module CLI
       return filename
     end
     
-    # TODO: This should probably be moved into its own class, into Util, or into Sifter?
-    def parse_sift_datetime(value)
-      value = Util.reduce_space(value).strip() # Don't use unspace_web_str(), want spaces for formats
-      value = value.split('...',2)
-      
-      check_empty_opt(:datetime,nil) if value.empty?() # For ''
-      
-      # Make a "to" and a "from" date time range.
-      value << value[0].dup() if value.length == 1
-      
-      to_day = nil
-      to_hour = 23
-      to_minute = 59
-      to_month = 12
-      to_year = Util::MAX_SANE_YEAR
-      
-      value.each_with_index() do |v,i|
-        v = check_empty_opt(:datetime,v) # For '...', '12-25...', or '...12-25'
-        
-        has_day = false
-        has_hour = false
-        has_minute = false
-        has_month = false
-        has_year = false
-        
-        SIFT_DATETIME_FMTS.each_with_index() do |fmt,i|
-          begin
-            # If don't do this, "%d" values will be parsed using "%d %H".
-            #   It seems as though strptime() ignores space.
-            raise ArgumentError if !v.include?(' ') && fmt.include?(' ')
-            
-            # If don't do this, "%y" values will be parsed using "%d".
-            raise ArgumentError if fmt == '%d' && v.length > 2
-            
-            v = Time.strptime(v,fmt,&Util.method(:guess_year))
-            
-            has_day = fmt.include?('%d')
-            has_hour = fmt.include?('%H')
-            has_minute = fmt.include?('%M')
-            has_month = fmt.include?('%m')
-            has_year = fmt.include?('%Y')
-            
-            break # No problem; this format worked
-          rescue ArgumentError
-            # Out of formats.
-            raise if i >= (SIFT_DATETIME_FMTS.length - 1)
-          end
-        end
-        
-        # "From" date time.
-        if i == 0
-          # Set these so that "2012-7-4...7-9" will use the appropriate year
-          #   of "2012" for "7-9".
-          to_day = v.day if has_day
-          to_hour = v.hour if has_hour
-          to_minute = v.min if has_minute
-          to_month = v.month if has_month
-          to_year = v.year if has_year
-          
-          v = Time.new(
-            has_year ? v.year : Util::MIN_SANE_YEAR,
-            has_month ? v.month : 1,
-            has_day ? v.day : 1,
-            has_hour ? v.hour : 0,
-            has_minute ? v.min : 0
-          )
-        # "To" date time.
-        else
-          to_hour = v.hour if has_hour
-          to_minute = v.min if has_minute
-          to_month = v.month if has_month
-          to_year = v.year if has_year
-          
-          if has_day
-            to_day = v.day
-          # Nothing passed from the "from" date time?
-          elsif to_day.nil?()
-            # Last day of month.
-            to_day = Date.new(to_year,to_month,-1).day
-          end
-          
-          v = Time.new(to_year,to_month,to_day,to_hour,to_minute)
-        end
-        
-        value[i] = v
-      end
-      
-      return value
-    end
-    
     def run_sift_cmd(type)
       news_name = nil
       
@@ -364,6 +259,8 @@ module CLI
         sifter.put_csv!()
       when :htm,:html
         sifter.put_html!()
+      when :json
+        sifter.put_json!()
       when :yaml,:yml
         sifter.put_yaml!()
       else