nhkore 0.3.13 → 0.3.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +17 -1
  3. data/Gemfile.lock +41 -41
  4. data/README.md +4 -5
  5. data/Rakefile +5 -3
  6. data/lib/nhkore/app.rb +8 -19
  7. data/lib/nhkore/article.rb +5 -9
  8. data/lib/nhkore/article_scraper.rb +15 -14
  9. data/lib/nhkore/cleaner.rb +0 -12
  10. data/lib/nhkore/cli/fx_cmd.rb +0 -4
  11. data/lib/nhkore/cli/get_cmd.rb +0 -4
  12. data/lib/nhkore/cli/news_cmd.rb +29 -17
  13. data/lib/nhkore/cli/search_cmd.rb +45 -35
  14. data/lib/nhkore/cli/sift_cmd.rb +1 -5
  15. data/lib/nhkore/datetime_parser.rb +1 -5
  16. data/lib/nhkore/defn.rb +1 -5
  17. data/lib/nhkore/dict.rb +2 -5
  18. data/lib/nhkore/dict_scraper.rb +2 -6
  19. data/lib/nhkore/entry.rb +3 -9
  20. data/lib/nhkore/error.rb +1 -11
  21. data/lib/nhkore/fileable.rb +0 -4
  22. data/lib/nhkore/lib.rb +0 -3
  23. data/lib/nhkore/missingno.rb +2 -6
  24. data/lib/nhkore/news.rb +3 -15
  25. data/lib/nhkore/polisher.rb +0 -12
  26. data/lib/nhkore/scraper.rb +8 -5
  27. data/lib/nhkore/search_link.rb +9 -17
  28. data/lib/nhkore/search_scraper.rb +34 -24
  29. data/lib/nhkore/sifter.rb +7 -8
  30. data/lib/nhkore/splitter.rb +0 -18
  31. data/lib/nhkore/user_agents.rb +1 -4
  32. data/lib/nhkore/util.rb +0 -4
  33. data/lib/nhkore/variator.rb +0 -14
  34. data/lib/nhkore/version.rb +1 -1
  35. data/lib/nhkore/word.rb +0 -4
  36. data/lib/nhkore.rb +0 -5
  37. data/nhkore.gemspec +40 -37
  38. data/samples/looper.rb +0 -3
  39. metadata +24 -24
data/lib/nhkore/sifter.rb CHANGED
@@ -15,10 +15,6 @@ require 'nhkore/util'
15
15
 
16
16
 
17
17
  module NHKore
18
- ###
19
- # @author Jonathan Bradley Whited
20
- # @since 0.2.0
21
- ###
22
18
  class Sifter
23
19
  include Fileable
24
20
 
@@ -61,10 +57,8 @@ module NHKore
61
57
  end
62
58
 
63
59
  def build_rows(words)
64
- rows = []
65
-
66
- words.each do |word|
67
- rows << build_word_row(word)
60
+ rows = words.map do |word|
61
+ build_word_row(word)
68
62
  end
69
63
 
70
64
  return rows
@@ -336,6 +330,11 @@ module NHKore
336
330
  next if filter?(article)
337
331
 
338
332
  article.words.each_value do |word|
333
+ # TODO: Try to remove garbage data better.
334
+ next if word.word.length < 2
335
+ next if word.freq <= 1
336
+ next if word.word =~ /\p{Latin}|[[:digit:]]/
337
+
339
338
  master_article.add_word(word,use_freq: true)
340
339
  end
341
340
  end
data/lib/nhkore/splitter.rb CHANGED
@@ -13,10 +13,6 @@ require 'nhkore/util'
13
13
 
14
14
 
15
15
  module NHKore
16
- ###
17
- # @author Jonathan Bradley Whited
18
- # @since 0.2.0
19
- ###
20
16
  class Splitter
21
17
  def begin_split(str)
22
18
  return str
@@ -30,19 +26,12 @@ module NHKore
30
26
  end
31
27
  end
32
28
 
33
- ###
34
- # @author Jonathan Bradley Whited
35
- # @since 0.2.0
36
- ###
37
29
  class BasicSplitter < Splitter
38
30
  def end_split(str)
39
31
  return str.split(Util::NORMALIZE_STR_REGEX)
40
32
  end
41
33
  end
42
34
 
43
- ###
44
- # @since 0.2.0
45
- ###
46
35
  class BimyouSplitter < Splitter
47
36
  def initialize(*)
48
37
  require 'bimyou_segmenter'
@@ -55,9 +44,6 @@ module NHKore
55
44
  end
56
45
  end
57
46
 
58
- ###
59
- # @since 0.2.0
60
- ###
61
47
  class TinySplitter < Splitter
62
48
  attr_accessor :tiny
63
49
 
@@ -74,10 +60,6 @@ module NHKore
74
60
  end
75
61
  end
76
62
 
77
- ###
78
- # @author Jonathan Bradley Whited
79
- # @since 0.2.0
80
- ###
81
63
  class BestSplitter < BimyouSplitter
82
64
  end
83
65
  end
data/lib/nhkore/user_agents.rb CHANGED
@@ -38,9 +38,6 @@ module NHKore
38
38
  #
39
39
  # The gem is really old and had a lot of warnings, so decided to make this class.
40
40
  # Maybe I'll fork the gem and maintain a new version in the future...
41
- #
42
- # @author Jonathan Bradley Whited
43
- # @since 0.2.1
44
41
  ###
45
42
  class UserAgents
46
43
  attr_accessor :data
@@ -53,7 +50,7 @@ module NHKore
53
50
  # because we don't need all of the data in memory after getting just 1
54
51
  # sample, even though it's slower.
55
52
  def initialize
56
- super()
53
+ super
57
54
 
58
55
  # rubocop:disable all
59
56
  @data = [
data/lib/nhkore/util.rb CHANGED
@@ -16,10 +16,6 @@ require 'uri'
16
16
 
17
17
 
18
18
  module NHKore
19
- ###
20
- # @author Jonathan Bradley Whited
21
- # @since 0.2.0
22
- ###
23
19
  module Util
24
20
  CORE_DIR = 'core'
25
21
  WEB_DIR = 'web'
data/lib/nhkore/variator.rb CHANGED
@@ -10,10 +10,6 @@
10
10
 
11
11
 
12
12
  module NHKore
13
- ###
14
- # @author Jonathan Bradley Whited
15
- # @since 0.2.0
16
- ###
17
13
  class Variator
18
14
  def begin_variate(str)
19
15
  return str
@@ -27,10 +23,6 @@ module NHKore
27
23
  end
28
24
  end
29
25
 
30
- ###
31
- # @author Jonathan Bradley Whited
32
- # @since 0.2.0
33
- ###
34
26
  class BasicVariator < Variator
35
27
  def end_variate(str)
36
28
  return [] # No variations; don't return nil
@@ -41,8 +33,6 @@ module NHKore
41
33
  # Guesses a word's dictionary/plain form (辞書形).
42
34
  #
43
35
  # It doesn't work very well,but better than nothing...
44
- #
45
- # @since 0.2.0
46
36
  ###
47
37
  class DictFormVariator < Variator
48
38
  attr_accessor :deinflector
@@ -66,10 +56,6 @@ module NHKore
66
56
  end
67
57
  end
68
58
 
69
- ###
70
- # @author Jonathan Bradley Whited
71
- # @since 0.2.0
72
- ###
73
59
  class BestVariator < DictFormVariator
74
60
  end
75
61
  end
data/lib/nhkore/version.rb CHANGED
@@ -10,5 +10,5 @@
10
10
 
11
11
 
12
12
  module NHKore
13
- VERSION = '0.3.13'
13
+ VERSION = '0.3.16'
14
14
  end
data/lib/nhkore/word.rb CHANGED
@@ -16,10 +16,6 @@ require 'nhkore/util'
16
16
 
17
17
 
18
18
  module NHKore
19
- ###
20
- # @author Jonathan Bradley Whited
21
- # @since 0.1.0
22
- ###
23
19
  class Word
24
20
  attr_accessor :defn
25
21
  attr_accessor :eng
data/lib/nhkore.rb CHANGED
@@ -26,12 +26,7 @@ require 'nhkore/cli/search_cmd'
26
26
  require 'nhkore/cli/sift_cmd'
27
27
 
28
28
 
29
- ###
30
- # @author Jonathan Bradley Whited
31
- # @since 0.1.0
32
- ###
33
29
  module NHKore
34
- # @since 0.2.0
35
30
  def self.run(args=ARGV)
36
31
  app = App.new(args)
37
32
 
data/nhkore.gemspec CHANGED
@@ -26,48 +26,57 @@ Gem::Specification.new do |spec|
26
26
  #'mailing_list_uri' => '',
27
27
  }
28
28
 
29
+ spec.required_ruby_version = '>= 2.5'
29
30
  spec.requirements = [
30
31
  'Nokogiri: https://www.nokogiri.org/tutorials/installing_nokogiri.html',
31
32
  ]
32
33
 
33
- spec.required_ruby_version = '>= 2.5'
34
- spec.require_paths = ['lib']
35
- spec.bindir = 'bin'
36
- spec.executables = [spec.name]
34
+ spec.require_paths = ['lib']
35
+ spec.bindir = 'bin'
36
+ spec.executables = [spec.name]
37
+
38
+ spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
39
+ spec.rdoc_options = [
40
+ '--hyperlink-all','--show-hash',
41
+ '--title',"NHKore v#{NHKore::VERSION} Doc",
42
+ '--main','README.md',
43
+ ]
37
44
 
38
45
  spec.files = [
39
46
  Dir.glob(File.join("{#{spec.require_paths.join(',')}}",'**','*.{erb,rb}')),
40
47
  Dir.glob(File.join(spec.bindir,'*')),
41
48
  Dir.glob(File.join('{samples,test,yard}','**','*.{erb,rb}')),
42
49
  %W[ Gemfile Gemfile.lock #{spec.name}.gemspec Rakefile .yardopts ],
43
- %w[ CHANGELOG.md LICENSE.txt README.md ],
50
+ spec.extra_rdoc_files,
44
51
  ].flatten
45
52
 
46
- spec.add_runtime_dependency 'attr_bool' ,'~> 0.2' # For attr_accessor?/attr_reader?
47
- spec.add_runtime_dependency 'bimyou_segmenter' ,'~> 1.2' # For splitting Japanese sentences into words
48
- spec.add_runtime_dependency 'cri' ,'~> 2.15' # For CLI commands/options
49
- spec.add_runtime_dependency 'down' ,'~> 5.3' # For downloading files (GetCmd)
50
- spec.add_runtime_dependency 'highline' ,'~> 2.0' # For CLI input/output
51
- spec.add_runtime_dependency 'http-cookie' ,'~> 1.0' # For parsing/setting cookies (BingScraper/Scraper)
52
- spec.add_runtime_dependency 'japanese_deinflector' ,'~> 0.0' # For unconjugating Japanese words (plain/dictionary form)
53
- spec.add_runtime_dependency 'nokogiri' ,'~> 1.13' # For scraping/hacking
54
- spec.add_runtime_dependency 'psychgus' ,'~> 1.3' # For styling Psych YAML
55
- spec.add_runtime_dependency 'public_suffix' ,'~> 4.0' # For parsing URL domain names
56
- spec.add_runtime_dependency 'rainbow' ,'~> 3.1' # For CLI color output
57
- spec.add_runtime_dependency 'rss' ,'~> 0.2' # For scraping (BingScraper/Scraper)
58
- spec.add_runtime_dependency 'rubyzip' ,'~> 2.3' # For extracting Zip files (GetCmd)
59
- spec.add_runtime_dependency 'tiny_segmenter' ,'~> 0.0' # For splitting Japanese sentences into words
60
- spec.add_runtime_dependency 'tty-progressbar' ,'~> 0.18' # For CLI progress bars
61
- spec.add_runtime_dependency 'tty-spinner' ,'~> 0.9' # For CLI spinning progress
53
+ run_dep = spec.method(:add_runtime_dependency)
54
+ run_dep[ 'attr_bool' ,'~> 0.2' ] # attr_accessor?/attr_reader?.
55
+ run_dep[ 'bimyou_segmenter' ,'~> 1.2' ] # Splitting Japanese sentences into words.
56
+ run_dep[ 'cri' ,'~> 2.15' ] # CLI commands/options.
57
+ run_dep[ 'down' ,'~> 5.4' ] # Downloading files (GetCmd).
58
+ run_dep[ 'highline' ,'~> 3.1' ] # CLI input/output.
59
+ run_dep[ 'http-cookie' ,'~> 1.0' ] # Parsing/Setting cookies [(Bing)Scraper].
60
+ run_dep[ 'japanese_deinflector','~> 0.0' ] # Unconjugating Japanese words (dictionary form).
61
+ run_dep[ 'nokogiri' ,'~> 1.16' ] # Scraping/Hacking.
62
+ run_dep[ 'psychgus' ,'~> 1.3' ] # Styling Psych YAML.
63
+ run_dep[ 'public_suffix' ,'~> 6.0' ] # Parsing URL domain names.
64
+ run_dep[ 'rainbow' ,'~> 3.1' ] # CLI color output.
65
+ run_dep[ 'rss' ,'~> 0.3' ] # Scraping [(Bing)Scraper].
66
+ run_dep[ 'rubyzip' ,'~> 2.3' ] # Extracting Zip files (GetCmd).
67
+ run_dep[ 'tiny_segmenter' ,'~> 0.0' ] # Splitting Japanese sentences into words.
68
+ run_dep[ 'tty-progressbar' ,'~> 0.18' ] # CLI progress bars.
69
+ run_dep[ 'tty-spinner' ,'~> 0.9' ] # CLI spinning progress.
62
70
 
63
- spec.add_development_dependency 'bundler' ,'~> 2.3'
64
- spec.add_development_dependency 'minitest' ,'~> 5.15'
65
- spec.add_development_dependency 'rake' ,'~> 13.0'
66
- spec.add_development_dependency 'raketeer' ,'~> 0.2' # For extra Rake tasks
67
- spec.add_development_dependency 'rdoc' ,'~> 6.4' # For YARDoc RDoc (*.rb)
68
- spec.add_development_dependency 'redcarpet' ,'~> 3.5' # For YARDoc Markdown (*.md)
69
- spec.add_development_dependency 'yard' ,'~> 0.9' # For documentation
70
- spec.add_development_dependency 'yard_ghurt','~> 1.2' # For extra YARDoc Rake tasks
71
+ dev_dep = spec.method(:add_development_dependency)
72
+ dev_dep[ 'bundler' ,'~> 2.5' ]
73
+ dev_dep[ 'minitest' ,'~> 5.25' ]
74
+ dev_dep[ 'rake' ,'~> 13.2' ]
75
+ dev_dep[ 'raketeer' ,'~> 0.2' ] # Extra Rake tasks.
76
+ dev_dep[ 'rdoc' ,'~> 6.7' ] # YARDoc RDoc (*.rb).
77
+ dev_dep[ 'redcarpet' ,'~> 3.6' ] # YARDoc Markdown (*.md).
78
+ dev_dep[ 'yard' ,'~> 0.9' ] # Doc.
79
+ dev_dep[ 'yard_ghurt','~> 1.2' ] # Extra YARDoc Rake tasks.
71
80
 
72
81
  spec.post_install_message = <<~MSG
73
82
  +=============================================================================+
@@ -81,13 +90,7 @@ Gem::Specification.new do |spec|
81
90
  | Changelog: #{spec.metadata['changelog_uri']}
82
91
  +=============================================================================+
83
92
  MSG
84
- #puts spec.post_install_message.split("\n").map(&:length).max
85
-
86
- spec.extra_rdoc_files = %w[ CHANGELOG.md LICENSE.txt README.md ]
87
93
 
88
- spec.rdoc_options = [
89
- '--hyperlink-all','--show-hash',
90
- '--title',"NHKore v#{NHKore::VERSION} Doc",
91
- '--main','README.md',
92
- ]
94
+ # Uncomment to see max line length:
95
+ #puts spec.post_install_message.split("\n").map(&:length).max
93
96
  end
data/samples/looper.rb CHANGED
@@ -13,9 +13,6 @@
13
13
  ###
14
14
  # If you run this script, be aware that it uses the +-F+ force option
15
15
  # (which overwrites files without prompting).
16
- #
17
- # @author Jonathan Bradley Whited
18
- # @since 0.3.2
19
16
  ###
20
17
 
21
18
  case ARGV[0]
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nhkore
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.13
4
+ version: 0.3.16
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan Bradley Whited
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-04-27 00:00:00.000000000 Z
11
+ date: 2024-08-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: attr_bool
@@ -58,28 +58,28 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '5.3'
61
+ version: '5.4'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '5.3'
68
+ version: '5.4'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: highline
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '2.0'
75
+ version: '3.1'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '2.0'
82
+ version: '3.1'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: http-cookie
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '1.13'
117
+ version: '1.16'
118
118
  type: :runtime
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '1.13'
124
+ version: '1.16'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: psychgus
127
127
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: '4.0'
145
+ version: '6.0'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: '4.0'
152
+ version: '6.0'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: rainbow
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,14 @@ dependencies:
170
170
  requirements:
171
171
  - - "~>"
172
172
  - !ruby/object:Gem::Version
173
- version: '0.2'
173
+ version: '0.3'
174
174
  type: :runtime
175
175
  prerelease: false
176
176
  version_requirements: !ruby/object:Gem::Requirement
177
177
  requirements:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
- version: '0.2'
180
+ version: '0.3'
181
181
  - !ruby/object:Gem::Dependency
182
182
  name: rubyzip
183
183
  requirement: !ruby/object:Gem::Requirement
@@ -240,42 +240,42 @@ dependencies:
240
240
  requirements:
241
241
  - - "~>"
242
242
  - !ruby/object:Gem::Version
243
- version: '2.3'
243
+ version: '2.5'
244
244
  type: :development
245
245
  prerelease: false
246
246
  version_requirements: !ruby/object:Gem::Requirement
247
247
  requirements:
248
248
  - - "~>"
249
249
  - !ruby/object:Gem::Version
250
- version: '2.3'
250
+ version: '2.5'
251
251
  - !ruby/object:Gem::Dependency
252
252
  name: minitest
253
253
  requirement: !ruby/object:Gem::Requirement
254
254
  requirements:
255
255
  - - "~>"
256
256
  - !ruby/object:Gem::Version
257
- version: '5.15'
257
+ version: '5.25'
258
258
  type: :development
259
259
  prerelease: false
260
260
  version_requirements: !ruby/object:Gem::Requirement
261
261
  requirements:
262
262
  - - "~>"
263
263
  - !ruby/object:Gem::Version
264
- version: '5.15'
264
+ version: '5.25'
265
265
  - !ruby/object:Gem::Dependency
266
266
  name: rake
267
267
  requirement: !ruby/object:Gem::Requirement
268
268
  requirements:
269
269
  - - "~>"
270
270
  - !ruby/object:Gem::Version
271
- version: '13.0'
271
+ version: '13.2'
272
272
  type: :development
273
273
  prerelease: false
274
274
  version_requirements: !ruby/object:Gem::Requirement
275
275
  requirements:
276
276
  - - "~>"
277
277
  - !ruby/object:Gem::Version
278
- version: '13.0'
278
+ version: '13.2'
279
279
  - !ruby/object:Gem::Dependency
280
280
  name: raketeer
281
281
  requirement: !ruby/object:Gem::Requirement
@@ -296,28 +296,28 @@ dependencies:
296
296
  requirements:
297
297
  - - "~>"
298
298
  - !ruby/object:Gem::Version
299
- version: '6.4'
299
+ version: '6.7'
300
300
  type: :development
301
301
  prerelease: false
302
302
  version_requirements: !ruby/object:Gem::Requirement
303
303
  requirements:
304
304
  - - "~>"
305
305
  - !ruby/object:Gem::Version
306
- version: '6.4'
306
+ version: '6.7'
307
307
  - !ruby/object:Gem::Dependency
308
308
  name: redcarpet
309
309
  requirement: !ruby/object:Gem::Requirement
310
310
  requirements:
311
311
  - - "~>"
312
312
  - !ruby/object:Gem::Version
313
- version: '3.5'
313
+ version: '3.6'
314
314
  type: :development
315
315
  prerelease: false
316
316
  version_requirements: !ruby/object:Gem::Requirement
317
317
  requirements:
318
318
  - - "~>"
319
319
  - !ruby/object:Gem::Version
320
- version: '3.5'
320
+ version: '3.6'
321
321
  - !ruby/object:Gem::Dependency
322
322
  name: yard
323
323
  requirement: !ruby/object:Gem::Requirement
@@ -412,7 +412,7 @@ metadata:
412
412
  changelog_uri: https://github.com/esotericpig/nhkore/blob/master/CHANGELOG.md
413
413
  post_install_message: |
414
414
  +=============================================================================+
415
- | NHKore v0.3.13
415
+ | NHKore v0.3.16
416
416
  |
417
417
  | You can now use [nhkore] on the command line.
418
418
  |
@@ -425,7 +425,7 @@ rdoc_options:
425
425
  - "--hyperlink-all"
426
426
  - "--show-hash"
427
427
  - "--title"
428
- - NHKore v0.3.13 Doc
428
+ - NHKore v0.3.16 Doc
429
429
  - "--main"
430
430
  - README.md
431
431
  require_paths: