nhkore 0.3.2 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf151c3859812632f09b1a464164f31bb0ce050f37ed7e7377f76265571ebd41
-  data.tar.gz: 1f3ee801e7557731cae4aeacd3f18fea4d7f33ac65b6ec77511a7d3d8f17856a
+  metadata.gz: 2b9464ae2a62f0c9cc797f2f70028d2b7afd6f0677a431c54e5453690175ca29
+  data.tar.gz: 577987179a9001926629f1efd8e8d39fb61ad62543d24ea1caa4f1fe063fd1a4
 SHA512:
-  metadata.gz: 7e7d0d5b805ad6fa4312e8be26f3115dff18665b3762073c56db3a7a6a343a3ee6a05e47889e0abf7b62df3bb84cf5c977fce3efdfeb8a65c7bcff8167839d35
-  data.tar.gz: 957bc3da8492310d287a8947b9080f8be417f0874c3226db4f0bb63d020bee06c51a3da81c1fa3f779de22d354a32ab4cf41fc6f3018840774c31fd7060fbec3
+  metadata.gz: '09d90011d4d641ea54c9dd7ebc8fd95efc8f7e68211e4322c1f3294e15a21303147de1eea2532694d5a01caaaf3c73f9a5172479193113be86b5b7a9fd08b910'
+  data.tar.gz: 65512547e6ee13503b345402e2eb1ba799e492131975f518cc96576a684bc97d48efb0f9c22787518f8dc62998e44fba1089ca2a1687b98e0533a489091e61c1

data/.yardopts

@@ -0,0 +1,3 @@
+--files 'CHANGELOG.md,LICENSE.txt'
+--protected
+--readme 'README.md'

data/CHANGELOG.md

@@ -2,7 +2,75 @@
 
 Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.2...master)
+## [[Unreleased]](https://github.com/esotericpig/nhkore/compare/v0.3.7...HEAD)
+
+## [v0.3.7] - 2020-11-07
+
+### Changed
+- Updated Gem `attr_bool` to v0.2
+- Changed upper-case *'-V'* flag for *version* to be a lower-case *'-v'*
+    - Seems like a lot of apps/people expect this
+- Refactored/Formatted some code
+    - *nhkore.gemspec* especially
+- Added *samples/*, *Gemfile.lock*, and *.yardopts* to the files in *nhkore.gemspec*
+
+### Fixed
+- ArticleScraper
+    - Fixed to accept text nodes that have Kanji, due to bad article:
+        - https://www3.nhk.or.jp/news/easy/k10012639271000/k10012639271000.html
+            - `第３のビール` should have HTML ruby tags around *第*
+
+## [v0.3.6] - 2020-08-18
+
+### Added
+- `update_showcase` Rake task for development & personal site (GitHub Page)
+    - `$ bundle exec rake update_showcase`
+
+### Changed
+- Updated Gems
+
+### Fixed
+- ArticleScraper for title for specific site
+    - https://www3.nhk.or.jp/news/easy/article/disaster_earthquake_illust.html
+- Ignored `/cgi2.*enqform/` URLs from SearchScraper (Bing)
+- Added more detail to dictionary error in ArticleScraper
+
+## [v0.3.5] - 2020-05-04
+
+### Added
+- Added check for environment var `NO_COLOR`
+    - [https://no-color.org/](https://no-color.org/)
+
+### Fixed
+- Fixed URLs stored in YAML data to always be of type String (not URI)
+    - This initially caused a problem in DictScraper.parse_url() from ArticleScraper, but fixed it for all data
+
+## [v0.3.4] - 2020-04-25
+
+### Added
+- DatetimeParser
+    - Extracted from SiftCmd into its own class
+    - Fixed some minor logic bugs from the old code
+    - Added new feature where 1 range can be empty:
+        - `sift ez -d '...2019'` (from = 1924)
+        - `sift ez -d '2019...'` (to = current year)
+        - `sift ez -d '...'` (still an error)
+- Added `update_core` rake task for dev
+    - Makes pushing a new release much easier
+    - See *Hacking.Releasing* section in *README*
+
+### Fixed
+- SiftCmd `parse_sift_datetime()` for `-d/--datetime` option
+    - Didn't work exactly right (as written in *README*) for some special inputs:
+        - `-d '2019...3'`
+        - `-d '3-3'`
+        - `-d '3'`
+
+## [v0.3.3] - 2020-04-23
+
+### Added
+- Added JSON support to Sifter & SiftCmd.
+- Added use of `attr_bool` Gem for `attr_accessor?` & `attr_reader?`.
 
 ## [v0.3.2] - 2020-04-22
 
@@ -23,10 +91,7 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - In filter_by_datetime(), renamed keyword args `from_filter,to_filter` to simply `from,to`.
 
 ### Fixed
-- Reduced load time of app from ~1s to 0.3~0.5s.
-    - Moved many `require '...'` statements into methods.
-    - It looks ugly & is not a good coding practice, but a necessary evil.
-    - Load time is still pretty slow (but a lot better!).
+- Reduced load time of app a tiny bit more (see v0.3.1 for details).
 - ArticleScraper
     - Renamed `mode` param to `strict`. `mode` was overshadowing File.open()'s in Scraper.
 
@@ -40,7 +105,10 @@ Format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - Changed `dir_str?()` and `filename_str?()` to check any slash. Previously, it only checked the slash for your system. But now on both Windows & Linux, it will check for both `/` & `\`.
 
 ### Fixed
-- Reduced load time of app from ~1s to ~0.3-5s by moving some requires into methods.
+- Reduced load time of app from about 1s to about 0.3-0.5s.
+    - Moved many `require '...'` statements into methods.
+    - It looks ugly & is not good coding practice, but a necessary evil.
+    - Load time is still pretty slow (but a lot better!).
 - BingScraper
     - Fixed possible RSS infinite loop.
 

data/Gemfile.lock

@@ -0,0 +1,86 @@
+PATH
+  remote: .
+  specs:
+    nhkore (0.3.7)
+      attr_bool (~> 0.2)
+      bimyou_segmenter (~> 1.2)
+      cri (~> 2.15)
+      down (~> 5.1)
+      highline (~> 2.0)
+      http-cookie (~> 1.0)
+      japanese_deinflector (~> 0.0)
+      nokogiri (~> 1.10)
+      psychgus (~> 1.3)
+      public_suffix (~> 4.0)
+      rainbow (~> 3.0)
+      rubyzip (~> 2.3)
+      tiny_segmenter (~> 0.0)
+      tty-progressbar (~> 0.17)
+      tty-spinner (~> 0.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    attr_bool (0.2.1)
+    bimyou_segmenter (1.2.0)
+    cri (2.15.10)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    down (5.2.0)
+      addressable (~> 2.5)
+    highline (2.0.3)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    japanese_deinflector (0.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.14.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    psych (3.2.0)
+    psychgus (1.3.3)
+      psych (>= 3.0)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    raketeer (0.2.9)
+      rake
+    rdoc (6.2.1)
+    redcarpet (3.5.0)
+    rubyzip (2.3.0)
+    strings-ansi (0.1.0)
+    tiny_segmenter (0.0.6)
+    tty-cursor (0.7.1)
+    tty-progressbar (0.17.0)
+      strings-ansi (~> 0.1.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.7)
+      unicode-display_width (~> 1.6)
+    tty-screen (0.8.1)
+    tty-spinner (0.9.3)
+      tty-cursor (~> 0.7)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
+    yard (0.9.25)
+    yard_ghurt (1.2.0)
+      rake
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.1)
+  minitest (~> 5.14)
+  nhkore!
+  rake (~> 13.0)
+  raketeer (~> 0.2)
+  rdoc (~> 6.2)
+  redcarpet (~> 3.5)
+  yard (~> 0.9)
+  yard_ghurt (~> 1.2)
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -26,6 +26,8 @@ This is similar to a [core word/vocabulary list](https://www.fluentin3months.com
         - [News Command](#news-command-)
 - [Using the Library](#using-the-library-)
 - [Hacking](#hacking-)
+    - [Updating](#updating-)
+    - [Releasing](#releasing-)
 - [License](#license-)
 
 ## For Non-Power Users [^](#contents)
@@ -110,11 +112,12 @@ Example usage:
 
 `$ nhkore -t 300 -m 10 news -D -L -M -d '2011-03-07 06:30' easy -u 'https://www3.nhk.or.jp/news/easy/tsunamikeihou/index.html'`
 
-Now that the data from the article has been scraped, you can generate a CSV/HTML/YAML file of the words ordered by frequency:
+Now that the data from the article has been scraped, you can generate a CSV/HTML/JSON/YAML file of the words ordered by frequency:
 
 ```
 $ nhkore sift easy -e csv
 $ nhkore sift easy -e html
+$ nhkore sift easy -e json
 $ nhkore sift easy -e yml
 ```
 
@@ -154,11 +157,11 @@ After obtaining the scraped data, you can `sift` all of the data (or select data
 | --- | --- |
 | CSV | For uploading to a flashcard website (e.g., Memrise, Anki, Buffl) after changing the data appropriately. |
 | HTML | For comfortable viewing in a web browser or for sharing. |
-| YAML | For developers to automatically add translations or to manipulate the data in some other way programmatically. |
+| YAML/JSON | For developers to automatically add translations or to manipulate the data in some other way programmatically. |
 
 The data is sorted by frequency in descending order (i.e., most frequent words first).
 
-If you wish to sort/arrange the data in some other way, CSV editors (e.g., LibreOffice, WPS Office, Microsoft Office) can do this easily and efficiently, or if you are code-savvy, you can programmatically manipulate the CSV/YAML/HTML file.
+If you wish to sort/arrange the data in some other way, CSV editors (e.g., LibreOffice, WPS Office, Microsoft Office) can do this easily and efficiently, or if you are code-savvy, you can programmatically manipulate the CSV/YAML/JSON/HTML file.
 
 The defaults will sift all of the data into a CSV file, which may not be what you want:
 
@@ -203,7 +206,7 @@ You can save the data to a different format using one of these options:
 
 ```
 -e --ext=<value>    type of file (extension) to save;
-                    valid options: [csv, htm, html, yaml, yml];
+                    valid options: [csv, htm, html, json, yaml, yml];
                     not needed if you specify a file extension with
                     the '--out' option: '--out sift.html'
                     (default: csv)
@@ -524,7 +527,7 @@ doc = ss.html_doc()
 doc.css('a').each() do |anchor|
   link = anchor['href']
   
-  next if ss.ignore_link?(link)
+  next if ss.ignore_link?(link,cleaned: false)
   
   if link.include?('https://www3.nhk')
     puts link
@@ -563,6 +566,7 @@ end
 
 ```Ruby
 require 'nhkore/article_scraper'
+require 'time'
 
 as = NHKore::ArticleScraper.new(
   'https://www3.nhk.or.jp/news/easy/k10011862381000/k10011862381000.html',
@@ -686,6 +690,7 @@ end
 `Sifter` will sift & sort the `News` data into a single file. The data is sorted by frequency in descending order (i.e., most frequent words first).
 
 ```Ruby
+require 'nhkore/datetime_parser'
 require 'nhkore/news'
 require 'nhkore/sifter'
 require 'time'
@@ -697,23 +702,26 @@ sifter = NHKore::Sifter.new(news)
 sifter.caption = 'Sakura Fields Forever!'
 
 # Filter the data.
-#sifter.filter_by_datetime(Time.new(2019,12,5))
+sifter.filter_by_datetime(NHKore::DatetimeParser.parse_range('2019-12-4...7'))
+sifter.filter_by_datetime([Time.new(2019,12,4),Time.new(2019,12,7)])
 sifter.filter_by_datetime(
   from: Time.new(2019,12,4),to: Time.new(2019,12,7)
 )
 sifter.filter_by_title('桜')
 sifter.filter_by_url('k100')
 
-# Ignore (or blank out) certain columns from the output.
+# Ignore certain columns from the output.
 sifter.ignore(:defn)
 sifter.ignore(:eng)
 
-# An array of the filtered & sorted words.
-words = sifter.sift()
+# An array of the sifted words.
+words = sifter.sift() # Filtered & Sorted array of Word
+rows  = sifter.build_rows(words) # Ignored array of array
 
 # Choose the file format.
 #sifter.put_csv!()
 #sifter.put_html!()
+#sifter.put_json!()
 sifter.put_yaml!()
 
 # Save to a file.
@@ -724,13 +732,14 @@ if !File.exist?(file)
 end
 ```
 
-### Util & UserAgents
+### Util, UserAgents, & DatetimeParser
 
 These provide a variety of useful methods/constants.
 
 Here are some of the most useful ones:
 
 ```Ruby
+require 'nhkore/datetime_parser'
 require 'nhkore/user_agents'
 require 'nhkore/util'
 
@@ -756,14 +765,16 @@ puts
 puts '========'
 puts '[ Time ]'
 puts '========'
-puts "JST now:   #{Util.jst_now}"
+puts "JST now:   #{Util.jst_now()}"
 # Drops in JST_OFFSET, does not change hour/min.
 puts "JST time:  #{Util.jst_time(Time.now)}"
 puts "JST year:  #{Util::JST_YEAR}"
 puts "1999 sane? #{Util.sane_year?(1999)}" # true
 puts "1776 sane? #{Util.sane_year?(1776)}" # false
-puts "Guess 5:   #{Util.guess_year(5)}"    # 2005
-puts "Guess 99:  #{Util.guess_year(99)}"   # 1999
+puts "Guess 5:   #{DatetimeParser.guess_year(5)}"  # 2005
+puts "Guess 99:  #{DatetimeParser.guess_year(99)}" # 1999
+# => [2020-12-01 00:00:00 +0900, 2020-12-31 23:59:59 +0900]
+puts "Parse:     #{DatetimeParser.parse_range('2020-12')}"
 puts
 puts "JST timezone offset:        #{Util::JST_OFFSET}"
 puts "JST timezone offset hour:   #{Util::JST_OFFSET_HOUR}"
@@ -789,9 +800,9 @@ def fmt_jpn()
 end
 
 puts "          #{fmt_jpn{|x| x}}"
-puts "Hiragana? #{fmt_jpn{|x| !!Util.hiragana?(x)}}"
-puts "Kana?     #{fmt_jpn{|x| !!Util.kana?(x)}}"
-puts "Kanji?    #{fmt_jpn{|x| !!Util.kanji?(x)}}"
+puts "Hiragana? #{fmt_jpn{|x| Util.hiragana?(x)}}"
+puts "Kana?     #{fmt_jpn{|x| Util.kana?(x)}}"
+puts "Kanji?    #{fmt_jpn{|x| Util.kanji?(x)}}"
 puts "Reduce:   #{Util.reduce_jpn_space("'     '")}"
 puts
 
@@ -839,9 +850,29 @@ You can make some changes/fixes to the code and then install your local version:
 
 `$ bundle exec rake install:local`
 
-### Releasing/Publishing
+### Updating [^](#contents)
+
+This will update *core/* for you:
+
+`$ bundle exec rake update_core`
+
+### Releasing [^](#contents)
+
+1. Update *CHANGELOG.md*, *version.rb*, & *Gemfile.lock*
+    - *Raketary*: `$ raketary bump -v`
+    - Run: `$ bundle update`
+2. Run: `$ bundle exec rake update_core`
+3. Run: `$ bundle exec rake clobber pkg_core`
+4. Create a new release & tag
+    - Add `pkg/nhkore-core.zip`
+5. Run: `$ git pull`
+6. Upload GitHub package
+    - *Raketary*: `$ raketary github_pkg`
+7. Run: `$ bundle exec rake release`
+
+Releasing new HTML file for website:
 
-`$ bundle exec rake release`
+1. `$ bundle exec rake update_showcase`
 
 ## License [^](#contents)
 

data/Rakefile

@@ -35,7 +35,7 @@ require 'nhkore/version'
 
 PKG_DIR = 'pkg'
 
-CLEAN.exclude('.git/','stock/')
+CLEAN.exclude('{.git,core,stock}/**/*')
 CLOBBER.include('doc/',File.join(PKG_DIR,''))
 
 
@@ -49,7 +49,7 @@ desc "Package '#{File.join(NHKore::Util::CORE_DIR,'')}' data as a Zip file into
 task :pkg_core do |task|
   mkdir_p PKG_DIR
   
-  pattern = File.join(NHKore::Util::CORE_DIR,'*.{csv,html,yml}')
+  pattern = File.join(NHKore::Util::CORE_DIR,'*.{csv,html,json,yml}')
   zip_file = File.join(PKG_DIR,'nhkore-core.zip')
   
   sh 'zip','-9rv',zip_file,*Dir.glob(pattern).sort()
@@ -59,17 +59,57 @@ Rake::TestTask.new() do |task|
   task.libs = ['lib','test']
   task.pattern = File.join('test','**','*_test.rb')
   task.description += ": '#{task.pattern}'"
-  task.verbose = true
+  task.verbose = false
   task.warning = true
 end
 
-YARD::Rake::YardocTask.new() do |task|
-  task.files = [File.join('lib','**','*.rb')]
+# If you need to run a part after the 1st part,
+# just type 'n' to not overwrite the file and then 'y' for continue.
+desc "Update '#{File.join(NHKore::Util::CORE_DIR,'')}' files for release"
+task :update_core do |task|
+  require 'highline'
+  
+  CONTINUE_MSG = "\nContinue (y/n)? "
+  
+  cmd = ['ruby','-w','./lib/nhkore.rb','-t','300','-m','10']
+  hl = HighLine.new()
+  
+  next unless sh(*cmd,'se','ez','bing')
+  next unless hl.agree(CONTINUE_MSG)
+  puts
   
-  task.options += ['--files','CHANGELOG.md,LICENSE.txt']
-  task.options += ['--readme','README.md']
+  next unless sh(*cmd,'news','-s','100','ez')
+  next unless hl.agree(CONTINUE_MSG)
+  puts
+  
+  next unless sh(*cmd,'sift','-e','csv' ,'ez')
+  next unless sh(*cmd,'sift','-e','html','ez')
+  next unless sh(*cmd,'sift','-e','json','ez')
+  next unless sh(*cmd,'sift','-e','yml' ,'ez')
+end
+
+# @since 0.3.6
+desc 'Update showcase file for release'
+task :update_showcase do |task|
+  require 'highline'
   
-  task.options << '--protected' # Show protected methods
+  SHOWCASE_FILE = File.join('.','nhkore-ez.html')
+  
+  hl = HighLine.new()
+  
+  next unless sh('ruby','-w','./lib/nhkore.rb',
+    'sift','ez','--no-eng',
+    '--out',SHOWCASE_FILE,
+  )
+  
+  next unless hl.agree("\nMove the file (y/n)? ")
+  puts
+  next unless sh('mv','-iv',SHOWCASE_FILE,
+    File.join('..','esotericpig.github.io','showcase',''),
+  )
+end
+
+YARD::Rake::YardocTask.new() do |task|
   task.options += ['--template-path',File.join('yard','templates')]
   task.options += ['--title',"NHKore v#{NHKore::VERSION} Doc"]
 end