nhkore 0.3.6 → 0.3.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.yardopts +3 -0
- data/CHANGELOG.md +63 -2
- data/Gemfile +0 -18
- data/Gemfile.lock +89 -0
- data/README.md +36 -30
- data/Rakefile +38 -52
- data/bin/nhkore +4 -15
- data/lib/nhkore.rb +8 -20
- data/lib/nhkore/app.rb +236 -236
- data/lib/nhkore/article.rb +39 -53
- data/lib/nhkore/article_scraper.rb +301 -287
- data/lib/nhkore/cleaner.rb +20 -32
- data/lib/nhkore/cli/fx_cmd.rb +41 -53
- data/lib/nhkore/cli/get_cmd.rb +59 -70
- data/lib/nhkore/cli/news_cmd.rb +143 -153
- data/lib/nhkore/cli/search_cmd.rb +108 -118
- data/lib/nhkore/cli/sift_cmd.rb +109 -120
- data/lib/nhkore/datetime_parser.rb +89 -103
- data/lib/nhkore/defn.rb +48 -55
- data/lib/nhkore/dict.rb +26 -38
- data/lib/nhkore/dict_scraper.rb +31 -40
- data/lib/nhkore/entry.rb +43 -55
- data/lib/nhkore/error.rb +16 -21
- data/lib/nhkore/fileable.rb +10 -21
- data/lib/nhkore/lib.rb +5 -17
- data/lib/nhkore/missingno.rb +21 -33
- data/lib/nhkore/news.rb +58 -72
- data/lib/nhkore/polisher.rb +22 -34
- data/lib/nhkore/scraper.rb +75 -82
- data/lib/nhkore/search_link.rb +63 -75
- data/lib/nhkore/search_scraper.rb +89 -93
- data/lib/nhkore/sifter.rb +157 -171
- data/lib/nhkore/splitter.rb +19 -31
- data/lib/nhkore/user_agents.rb +28 -32
- data/lib/nhkore/util.rb +72 -84
- data/lib/nhkore/variator.rb +20 -32
- data/lib/nhkore/version.rb +4 -16
- data/lib/nhkore/word.rb +105 -99
- data/nhkore.gemspec +54 -65
- data/samples/looper.rb +71 -0
- data/test/nhkore/test_helper.rb +3 -15
- data/test/nhkore_test.rb +6 -18
- metadata +50 -28
data/bin/nhkore
CHANGED
@@ -4,24 +4,13 @@
|
|
4
4
|
|
5
5
|
#--
|
6
6
|
# This file is part of NHKore.
|
7
|
-
# Copyright (c) 2020 Jonathan Bradley Whited
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU Lesser General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# NHKore is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU Lesser General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU Lesser General Public License
|
20
|
-
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
|
7
|
+
# Copyright (c) 2020-2021 Jonathan Bradley Whited
|
8
|
+
#
|
9
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
21
10
|
#++
|
22
11
|
|
23
12
|
|
24
13
|
require 'nhkore'
|
25
14
|
|
26
15
|
|
27
|
-
NHKore.run
|
16
|
+
NHKore.run
|
data/lib/nhkore.rb
CHANGED
@@ -1,27 +1,15 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of NHKore.
|
7
|
-
# Copyright (c) 2020 Jonathan Bradley Whited
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU Lesser General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# NHKore is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU Lesser General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU Lesser General Public License
|
20
|
-
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2020-2021 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
24
|
-
TESTING = ($
|
12
|
+
TESTING = ($PROGRAM_NAME == __FILE__)
|
25
13
|
|
26
14
|
if TESTING
|
27
15
|
require 'rubygems'
|
@@ -39,16 +27,16 @@ require 'nhkore/cli/sift_cmd'
|
|
39
27
|
|
40
28
|
|
41
29
|
###
|
42
|
-
# @author Jonathan Bradley Whited
|
30
|
+
# @author Jonathan Bradley Whited
|
43
31
|
# @since 0.1.0
|
44
32
|
###
|
45
33
|
module NHKore
|
46
34
|
# @since 0.2.0
|
47
35
|
def self.run(args=ARGV)
|
48
36
|
app = App.new(args)
|
49
|
-
|
37
|
+
|
50
38
|
begin
|
51
|
-
app.run
|
39
|
+
app.run
|
52
40
|
rescue CLIError => e
|
53
41
|
puts "Error: #{e}"
|
54
42
|
exit 1
|
@@ -56,4 +44,4 @@ module NHKore
|
|
56
44
|
end
|
57
45
|
end
|
58
46
|
|
59
|
-
NHKore.run
|
47
|
+
NHKore.run if TESTING
|
data/lib/nhkore/app.rb
CHANGED
@@ -1,23 +1,11 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of NHKore.
|
7
|
-
# Copyright (c) 2020 Jonathan Bradley Whited
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU Lesser General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# NHKore is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU Lesser General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU Lesser General Public License
|
20
|
-
# along with NHKore. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2020-2021 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
@@ -40,32 +28,32 @@ require 'nhkore/cli/sift_cmd'
|
|
40
28
|
|
41
29
|
module NHKore
|
42
30
|
###
|
43
|
-
# @author Jonathan Bradley Whited
|
31
|
+
# @author Jonathan Bradley Whited
|
44
32
|
# @since 0.2.0
|
45
33
|
###
|
46
34
|
module CLI
|
47
35
|
end
|
48
|
-
|
36
|
+
|
49
37
|
###
|
50
38
|
# For disabling/enabling color output.
|
51
|
-
#
|
52
|
-
# @author Jonathan Bradley Whited
|
39
|
+
#
|
40
|
+
# @author Jonathan Bradley Whited
|
53
41
|
# @since 0.2.1
|
54
42
|
###
|
55
43
|
module CriColorExt
|
56
|
-
|
57
|
-
|
44
|
+
@color = true
|
45
|
+
|
58
46
|
def color=(color)
|
59
|
-
|
47
|
+
@color = color
|
60
48
|
end
|
61
|
-
|
49
|
+
|
62
50
|
def color?(io)
|
63
|
-
return
|
51
|
+
return @color
|
64
52
|
end
|
65
53
|
end
|
66
|
-
|
54
|
+
|
67
55
|
###
|
68
|
-
# @author Jonathan Bradley Whited
|
56
|
+
# @author Jonathan Bradley Whited
|
69
57
|
# @since 0.2.0
|
70
58
|
###
|
71
59
|
class App
|
@@ -74,21 +62,20 @@ module NHKore
|
|
74
62
|
include CLI::NewsCmd
|
75
63
|
include CLI::SearchCmd
|
76
64
|
include CLI::SiftCmd
|
77
|
-
|
65
|
+
|
78
66
|
NAME = 'nhkore'
|
79
|
-
|
67
|
+
|
80
68
|
DEFAULT_SLEEP_TIME = 0.1 # So that sites don't ban us (i.e., think we are human)
|
81
|
-
|
82
|
-
COLOR_OPTS = [
|
83
|
-
NO_COLOR_OPTS = [
|
84
|
-
|
69
|
+
|
70
|
+
COLOR_OPTS = %i[c color].freeze
|
71
|
+
NO_COLOR_OPTS = %i[C no-color].freeze
|
72
|
+
|
85
73
|
SPINNER_MSG = '[:spinner] :title:detail...'
|
86
74
|
CLASSIC_SPINNER = TTY::Spinner.new(SPINNER_MSG,format: :classic)
|
87
75
|
DEFAULT_SPINNER = TTY::Spinner.new(SPINNER_MSG,interval: 5,
|
88
76
|
frames: ['〜〜〜','日〜〜','日本〜','日本語'])
|
89
|
-
NO_SPINNER = {} # Still outputs status & stores tokens
|
90
77
|
NO_SPINNER_MSG = '%{title}%{detail}...'
|
91
|
-
|
78
|
+
|
92
79
|
attr_reader :cmd
|
93
80
|
attr_reader :cmd_args
|
94
81
|
attr_reader :cmd_opts
|
@@ -96,108 +83,108 @@ module NHKore
|
|
96
83
|
attr_accessor :scraper_kargs
|
97
84
|
attr_accessor :sleep_time
|
98
85
|
attr_accessor :spinner
|
99
|
-
|
86
|
+
|
100
87
|
def initialize(args=ARGV)
|
101
88
|
super()
|
102
|
-
|
89
|
+
|
103
90
|
@args = args
|
104
91
|
@cmd = nil
|
105
92
|
@cmd_args = nil
|
106
93
|
@cmd_opts = nil
|
107
|
-
@high = HighLine.new
|
108
|
-
@rainbow = Rainbow.new
|
94
|
+
@high = HighLine.new
|
95
|
+
@rainbow = Rainbow.new
|
109
96
|
@progress_bar = :default # [:default, :classic, :no]
|
110
97
|
@scraper_kargs = {}
|
111
98
|
@sleep_time = DEFAULT_SLEEP_TIME
|
112
99
|
@spinner = DEFAULT_SPINNER
|
113
|
-
|
114
|
-
autodetect_color
|
115
|
-
|
116
|
-
build_app_cmd
|
117
|
-
|
118
|
-
build_fx_cmd
|
119
|
-
build_get_cmd
|
120
|
-
build_news_cmd
|
121
|
-
build_search_cmd
|
122
|
-
build_sift_cmd
|
123
|
-
build_version_cmd
|
124
|
-
|
125
|
-
@app_cmd.add_command Cri::Command.new_basic_help
|
126
|
-
end
|
127
|
-
|
128
|
-
def autodetect_color
|
100
|
+
|
101
|
+
autodetect_color
|
102
|
+
|
103
|
+
build_app_cmd
|
104
|
+
|
105
|
+
build_fx_cmd
|
106
|
+
build_get_cmd
|
107
|
+
build_news_cmd
|
108
|
+
build_search_cmd
|
109
|
+
build_sift_cmd
|
110
|
+
build_version_cmd
|
111
|
+
|
112
|
+
@app_cmd.add_command Cri::Command.new_basic_help
|
113
|
+
end
|
114
|
+
|
115
|
+
def autodetect_color
|
129
116
|
Cri::Platform.singleton_class.prepend(CriColorExt)
|
130
|
-
|
117
|
+
|
131
118
|
color = nil # Must be nil, not true/false
|
132
|
-
|
133
|
-
if !@args.empty?
|
119
|
+
|
120
|
+
if !@args.empty?
|
134
121
|
# Kind of hacky, but necessary for Rainbow.
|
135
|
-
|
122
|
+
|
136
123
|
color_opts = opts_to_set(COLOR_OPTS)
|
137
124
|
no_color_opts = opts_to_set(NO_COLOR_OPTS)
|
138
|
-
|
139
|
-
@args.each
|
125
|
+
|
126
|
+
@args.each do |arg|
|
140
127
|
if color_opts.include?(arg)
|
141
128
|
color = true
|
142
129
|
break
|
143
130
|
end
|
144
|
-
|
131
|
+
|
145
132
|
if no_color_opts.include?(arg)
|
146
133
|
color = false
|
147
134
|
break
|
148
135
|
end
|
149
|
-
|
136
|
+
|
150
137
|
break if arg == '--'
|
151
138
|
end
|
152
139
|
end
|
153
|
-
|
154
|
-
if color.nil?
|
140
|
+
|
141
|
+
if color.nil?
|
155
142
|
# - https://no-color.org/
|
156
|
-
color = ($stdout.tty?
|
143
|
+
color = ($stdout.tty? && ENV['TERM'] != 'dumb' && !ENV.key?('NO_COLOR'))
|
157
144
|
end
|
158
|
-
|
145
|
+
|
159
146
|
enable_color(color)
|
160
147
|
end
|
161
|
-
|
162
|
-
def build_app_cmd
|
148
|
+
|
149
|
+
def build_app_cmd
|
163
150
|
app = self
|
164
|
-
|
165
|
-
@app_cmd = Cri::Command.define
|
151
|
+
|
152
|
+
@app_cmd = Cri::Command.define do
|
166
153
|
name NAME
|
167
154
|
usage "#{NAME} [OPTIONS] [COMMAND]..."
|
168
155
|
summary 'NHK News Web (Easy) scraper for Japanese language learners.'
|
169
|
-
|
170
|
-
description <<-
|
156
|
+
|
157
|
+
description <<-DESC
|
171
158
|
Scrapes NHK News Web (Easy) to create a list of each word and its
|
172
159
|
frequency (how many times it was used) for Japanese language learners.
|
173
|
-
|
160
|
+
|
174
161
|
This is similar to a core word/vocabulary list.
|
175
|
-
|
176
|
-
|
177
|
-
flag :s,:'classic-fx',<<-
|
162
|
+
DESC
|
163
|
+
|
164
|
+
flag :s,:'classic-fx',<<-DESC do |value,cmd|
|
178
165
|
use classic spinner/progress special effects (in case of no Unicode support) when running long tasks
|
179
|
-
|
166
|
+
DESC
|
180
167
|
app.progress_bar = :classic
|
181
168
|
app.spinner = CLASSIC_SPINNER
|
182
169
|
end
|
183
|
-
flag COLOR_OPTS[0],COLOR_OPTS[1]
|
170
|
+
flag COLOR_OPTS[0],COLOR_OPTS[1],"force color output (for commands like '| less -R')" do |value,cmd|
|
184
171
|
app.enable_color(true)
|
185
172
|
end
|
186
|
-
flag :n,:'dry-run',<<-
|
173
|
+
flag :n,:'dry-run',<<-DESC
|
187
174
|
do a dry run without making changes; do not write to files, create directories, etc.
|
188
|
-
|
175
|
+
DESC
|
189
176
|
# Big F because dangerous.
|
190
177
|
flag :F,:force,"force overwriting files, creating directories, etc. (don't prompt); dangerous!"
|
191
178
|
flag :h,:help,'show this help' do |value,cmd|
|
192
179
|
puts cmd.help
|
193
180
|
exit
|
194
181
|
end
|
195
|
-
option :m,:'max-retry',<<-
|
182
|
+
option :m,:'max-retry',<<-DESC,argument: :required,default: 3 do |value,cmd|
|
196
183
|
maximum number of times to retry URLs (-1 or integer >= 0)
|
197
|
-
|
198
|
-
value = value.to_i
|
184
|
+
DESC
|
185
|
+
value = value.to_i
|
199
186
|
value = nil if value < 0
|
200
|
-
|
187
|
+
|
201
188
|
app.scraper_kargs[:max_retries] = value
|
202
189
|
end
|
203
190
|
flag NO_COLOR_OPTS[0],NO_COLOR_OPTS[1],'disable color output' do |value,cmd|
|
@@ -205,81 +192,80 @@ module NHKore
|
|
205
192
|
end
|
206
193
|
flag :X,:'no-fx','disable spinner/progress special effects when running long tasks' do |value,cmd|
|
207
194
|
app.progress_bar = :no
|
208
|
-
app.spinner =
|
195
|
+
app.spinner = {} # Still outputs status & stores tokens
|
209
196
|
end
|
210
|
-
option :o,:'open-timeout',<<-
|
197
|
+
option :o,:'open-timeout',<<-DESC,argument: :required do |value,cmd|
|
211
198
|
seconds for URL open timeouts (-1 or decimal >= 0)
|
212
|
-
|
213
|
-
value = value.to_f
|
199
|
+
DESC
|
200
|
+
value = value.to_f
|
214
201
|
value = nil if value < 0.0
|
215
|
-
|
202
|
+
|
216
203
|
app.scraper_kargs[:open_timeout] = value
|
217
204
|
end
|
218
|
-
option :r,:'read-timeout',<<-
|
205
|
+
option :r,:'read-timeout',<<-DESC,argument: :required do |value,cmd|
|
219
206
|
seconds for URL read timeouts (-1 or decimal >= 0)
|
220
|
-
|
221
|
-
value = value.to_f
|
207
|
+
DESC
|
208
|
+
value = value.to_f
|
222
209
|
value = nil if value < 0.0
|
223
|
-
|
210
|
+
|
224
211
|
app.scraper_kargs[:read_timeout] = value
|
225
212
|
end
|
226
|
-
option :z,:sleep,<<-
|
213
|
+
option :z,:sleep,<<-DESC,argument: :required,default: DEFAULT_SLEEP_TIME do |value,cmd|
|
227
214
|
seconds to sleep per scrape (i.e., per page/article) so don't get banned (i.e., fake being human)
|
228
|
-
|
229
|
-
app.sleep_time = value.to_f
|
215
|
+
DESC
|
216
|
+
app.sleep_time = value.to_f
|
230
217
|
app.sleep_time = 0.0 if app.sleep_time < 0.0
|
231
218
|
end
|
232
|
-
option :t,:timeout,<<-
|
219
|
+
option :t,:timeout,<<-DESC,argument: :required do |value,cmd|
|
233
220
|
seconds for all URL timeouts: [open, read] (-1 or decimal >= 0)
|
234
|
-
|
235
|
-
value = value.to_f
|
221
|
+
DESC
|
222
|
+
value = value.to_f
|
236
223
|
value = nil if value < 0.0
|
237
|
-
|
224
|
+
|
238
225
|
app.scraper_kargs[:open_timeout] = value
|
239
226
|
app.scraper_kargs[:read_timeout] = value
|
240
227
|
end
|
241
|
-
option :u,:'user-agent',<<-
|
228
|
+
option :u,:'user-agent',<<-DESC,argument: :required do |value,cmd|
|
242
229
|
HTTP header field 'User-Agent' to use instead of a random one
|
243
|
-
|
230
|
+
DESC
|
244
231
|
value = app.check_empty_opt(:'user-agent',value)
|
245
|
-
|
232
|
+
|
246
233
|
app.scraper_kargs[:header] ||= {}
|
247
234
|
app.scraper_kargs[:header]['user-agent'] = value
|
248
235
|
end
|
249
|
-
|
250
|
-
|
251
|
-
app.show_version()
|
236
|
+
flag :v,:version,'show the version and exit' do |value,cmd|
|
237
|
+
app.show_version
|
252
238
|
exit
|
253
239
|
end
|
254
|
-
|
240
|
+
|
255
241
|
run do |opts,args,cmd|
|
256
242
|
puts cmd.help
|
257
243
|
end
|
258
244
|
end
|
259
245
|
end
|
260
|
-
|
246
|
+
|
261
247
|
def build_dir(opt_key,default_dir: '.')
|
262
248
|
# Protect against fat-fingering.
|
263
249
|
default_dir = Util.strip_web_str(default_dir)
|
264
|
-
dir = Util.strip_web_str(@cmd_opts[opt_key].to_s
|
265
|
-
|
266
|
-
dir = default_dir if dir.empty?
|
267
|
-
|
250
|
+
dir = Util.strip_web_str(@cmd_opts[opt_key].to_s)
|
251
|
+
|
252
|
+
dir = default_dir if dir.empty?
|
253
|
+
|
268
254
|
# '~' will expand to home, etc.
|
269
|
-
dir = File.expand_path(dir) unless dir.nil?
|
270
|
-
|
255
|
+
dir = File.expand_path(dir) unless dir.nil?
|
256
|
+
|
271
257
|
return (@cmd_opts[opt_key] = dir)
|
272
258
|
end
|
273
|
-
|
259
|
+
|
274
260
|
def build_file(opt_key,default_dir: '.',default_filename: '')
|
275
261
|
# Protect against fat-fingering.
|
276
262
|
default_dir = Util.strip_web_str(default_dir)
|
277
263
|
default_filename = Util.strip_web_str(default_filename)
|
278
|
-
file = Util.strip_web_str(@cmd_opts[opt_key].to_s
|
279
|
-
|
280
|
-
if file.empty?
|
264
|
+
file = Util.strip_web_str(@cmd_opts[opt_key].to_s)
|
265
|
+
|
266
|
+
if file.empty?
|
281
267
|
# Do not check default_dir.empty?().
|
282
|
-
if default_filename.empty?
|
268
|
+
if default_filename.empty?
|
283
269
|
file = nil # nil is very important for BingScraper.init()!
|
284
270
|
else
|
285
271
|
file = File.join(default_dir,default_filename)
|
@@ -294,347 +280,361 @@ module NHKore
|
|
294
280
|
end
|
295
281
|
# Else, passed in both: 'directory/file'
|
296
282
|
end
|
297
|
-
|
283
|
+
|
298
284
|
# '~' will expand to home, etc.
|
299
|
-
file = File.expand_path(file) unless file.nil?
|
300
|
-
|
285
|
+
file = File.expand_path(file) unless file.nil?
|
286
|
+
|
301
287
|
return (@cmd_opts[opt_key] = file)
|
302
288
|
end
|
303
|
-
|
289
|
+
|
304
290
|
def build_in_dir(opt_key,**kargs)
|
305
291
|
return build_dir(opt_key,**kargs)
|
306
292
|
end
|
307
|
-
|
293
|
+
|
308
294
|
def build_in_file(opt_key,**kargs)
|
309
295
|
return build_file(opt_key,**kargs)
|
310
296
|
end
|
311
|
-
|
297
|
+
|
312
298
|
def build_out_dir(opt_key,**kargs)
|
313
299
|
return build_dir(opt_key,**kargs)
|
314
300
|
end
|
315
|
-
|
301
|
+
|
316
302
|
def build_out_file(opt_key,**kargs)
|
317
303
|
return build_file(opt_key,**kargs)
|
318
304
|
end
|
319
|
-
|
305
|
+
|
320
306
|
def build_progress_bar(title,download: false,total: 100,type: @progress_bar,width: 33,**kargs)
|
321
307
|
case type
|
322
308
|
when :default,:classic
|
323
309
|
require 'tty-progressbar'
|
324
|
-
|
325
|
-
msg = "#{title} [:bar] :percent :eta".dup
|
310
|
+
|
311
|
+
msg = "#{title} [:bar] :percent :eta".dup
|
326
312
|
msg << ' :byte_rate/s' if download
|
327
|
-
|
313
|
+
|
328
314
|
return TTY::ProgressBar.new(msg,total: total,width: width,**kargs) do |config|
|
329
315
|
if type == :default
|
330
316
|
config.incomplete = '.'
|
331
317
|
config.complete = '/'
|
332
318
|
config.head = 'o'
|
333
319
|
end
|
334
|
-
|
320
|
+
|
335
321
|
#config.frequency = 5 # For a big download, set this
|
336
322
|
config.interval = 1 if download
|
337
323
|
end
|
338
324
|
end
|
339
|
-
|
325
|
+
|
340
326
|
# :no
|
341
327
|
return NoProgressBar.new(title,total: total,**kargs)
|
342
328
|
end
|
343
|
-
|
344
|
-
def build_version_cmd
|
329
|
+
|
330
|
+
def build_version_cmd
|
345
331
|
app = self
|
346
|
-
|
347
|
-
@version_cmd = @app_cmd.define_command
|
332
|
+
|
333
|
+
@version_cmd = @app_cmd.define_command do
|
348
334
|
name 'version'
|
349
335
|
usage 'version [OPTIONS] [COMMAND]...'
|
350
336
|
aliases :v
|
351
337
|
summary "Show the version and exit (aliases: #{app.color_alias('v')})"
|
352
|
-
|
338
|
+
|
353
339
|
run do |opts,args,cmd|
|
354
|
-
app.show_version
|
340
|
+
app.show_version
|
355
341
|
end
|
356
342
|
end
|
357
343
|
end
|
358
|
-
|
344
|
+
|
359
345
|
def check_empty_opt(key,value)
|
360
|
-
value = Util.strip_web_str(value) unless value.nil?
|
361
|
-
|
362
|
-
if value.nil?
|
346
|
+
value = Util.strip_web_str(value) unless value.nil?
|
347
|
+
|
348
|
+
if value.nil? || value.empty?
|
363
349
|
raise CLIError,"option[#{key}] cannot be empty[#{value}]"
|
364
350
|
end
|
365
|
-
|
351
|
+
|
366
352
|
return value
|
367
353
|
end
|
368
|
-
|
354
|
+
|
369
355
|
def check_in_file(opt_key,empty_ok: false)
|
370
356
|
in_file = @cmd_opts[opt_key]
|
371
|
-
|
357
|
+
|
372
358
|
if Util.empty_web_str?(in_file)
|
373
359
|
if !empty_ok
|
374
360
|
raise CLIError,"empty input path name[#{in_file}] in option[#{opt_key}]"
|
375
361
|
end
|
376
|
-
|
362
|
+
|
377
363
|
@cmd_opts[opt_key] = nil # nil is very important for BingScraper.init()!
|
378
|
-
|
364
|
+
|
379
365
|
return true
|
380
366
|
end
|
381
|
-
|
367
|
+
|
382
368
|
in_file = Util.strip_web_str(in_file)
|
383
|
-
|
369
|
+
|
384
370
|
if !File.exist?(in_file)
|
385
371
|
raise CLIError,"input file[#{in_file}] does not exist for option[#{opt_key}]"
|
386
372
|
end
|
387
|
-
|
373
|
+
|
388
374
|
if File.directory?(in_file)
|
389
375
|
raise CLIError,"input file[#{in_file}] cannot be a directory for option[#{opt_key}]"
|
390
376
|
end
|
391
|
-
|
377
|
+
|
392
378
|
return true
|
393
379
|
end
|
394
|
-
|
380
|
+
|
395
381
|
def check_out_dir(opt_key)
|
396
382
|
out_dir = @cmd_opts[opt_key]
|
397
|
-
|
383
|
+
|
398
384
|
if Util.empty_web_str?(out_dir)
|
399
385
|
raise CLIError,"empty output directory[#{out_dir}] in option[#{opt_key}]"
|
400
386
|
end
|
401
|
-
|
387
|
+
|
402
388
|
out_dir = Util.strip_web_str(out_dir)
|
403
|
-
|
389
|
+
|
404
390
|
if File.file?(out_dir)
|
405
391
|
raise CLIError,"output directory[#{out_dir}] cannot be a file for option[#{opt_key}]"
|
406
392
|
end
|
407
|
-
|
393
|
+
|
408
394
|
if @cmd_opts[:dry_run]
|
409
395
|
puts 'No changes written (dry run).'
|
410
396
|
puts "> #{out_dir}"
|
411
397
|
puts
|
412
|
-
|
398
|
+
|
413
399
|
return true
|
414
400
|
end
|
415
|
-
|
401
|
+
|
416
402
|
force = @cmd_opts[:force]
|
417
|
-
|
403
|
+
|
418
404
|
if !force && Dir.exist?(out_dir) && !Dir.empty?(out_dir)
|
419
405
|
puts 'Warning: output directory already exists with files!'
|
420
406
|
puts ' : Files inside of this directory may be overwritten!'
|
421
407
|
puts "> '#{out_dir}'"
|
422
|
-
|
408
|
+
|
423
409
|
return false unless @high.agree('Is this okay (yes/no)? ')
|
424
410
|
puts
|
425
411
|
end
|
426
|
-
|
412
|
+
|
427
413
|
if !Dir.exist?(out_dir)
|
428
414
|
if !force
|
429
415
|
puts 'Output directory does not exist.'
|
430
416
|
puts "> '#{out_dir}'"
|
431
|
-
|
417
|
+
|
432
418
|
return false unless @high.agree('Create this directory (yes/no)? ')
|
433
419
|
end
|
434
|
-
|
420
|
+
|
435
421
|
FileUtils.mkdir_p(out_dir,verbose: true)
|
436
422
|
puts
|
437
423
|
end
|
438
|
-
|
424
|
+
|
439
425
|
return true
|
440
426
|
end
|
441
|
-
|
427
|
+
|
442
428
|
def check_out_file(opt_key)
|
443
429
|
out_file = @cmd_opts[opt_key]
|
444
|
-
|
430
|
+
|
445
431
|
if Util.empty_web_str?(out_file)
|
446
432
|
raise CLIError,"empty output path name[#{out_file}] in option[#{opt_key}]"
|
447
433
|
end
|
448
|
-
|
434
|
+
|
449
435
|
out_file = Util.strip_web_str(out_file)
|
450
|
-
|
436
|
+
|
451
437
|
if File.directory?(out_file)
|
452
438
|
raise CLIError,"output file[#{out_file}] cannot be a directory for option[#{opt_key}]"
|
453
439
|
end
|
454
|
-
|
440
|
+
|
455
441
|
if @cmd_opts[:dry_run]
|
456
442
|
puts 'No changes written (dry run).'
|
457
443
|
puts "> #{out_file}"
|
458
444
|
puts
|
459
|
-
|
445
|
+
|
460
446
|
return true
|
461
447
|
end
|
462
|
-
|
448
|
+
|
463
449
|
force = @cmd_opts[:force]
|
464
450
|
out_dir = File.dirname(out_file)
|
465
|
-
|
451
|
+
|
466
452
|
if !force && File.exist?(out_file)
|
467
453
|
puts 'Warning: output file already exists!'
|
468
454
|
puts "> '#{out_file}'"
|
469
|
-
|
455
|
+
|
470
456
|
return false unless @high.agree('Overwrite this file (yes/no)? ')
|
471
457
|
puts
|
472
458
|
end
|
473
|
-
|
459
|
+
|
474
460
|
if !Dir.exist?(out_dir)
|
475
461
|
if !force
|
476
462
|
puts 'Output directory does not exist.'
|
477
463
|
puts "> '#{out_dir}'"
|
478
|
-
|
464
|
+
|
479
465
|
return false unless @high.agree('Create this directory (yes/no)? ')
|
480
466
|
end
|
481
|
-
|
467
|
+
|
482
468
|
FileUtils.mkdir_p(out_dir,verbose: true)
|
483
469
|
puts
|
484
470
|
end
|
485
|
-
|
471
|
+
|
486
472
|
return true
|
487
473
|
end
|
488
|
-
|
474
|
+
|
489
475
|
def color(str)
|
490
476
|
return @rainbow.wrap(str)
|
491
477
|
end
|
492
|
-
|
478
|
+
|
493
479
|
def color_alias(str)
|
494
480
|
return color(str).green
|
495
481
|
end
|
496
|
-
|
482
|
+
|
497
483
|
def enable_color(enabled)
|
498
484
|
Cri::Platform.color = enabled
|
499
485
|
@rainbow.enabled = enabled
|
500
486
|
end
|
501
|
-
|
487
|
+
|
502
488
|
def opts_to_set(ary)
|
503
|
-
set = Set.new
|
504
|
-
|
505
|
-
set.add("-#{ary[0]
|
506
|
-
set.add("--#{ary[1]
|
507
|
-
|
489
|
+
set = Set.new
|
490
|
+
|
491
|
+
set.add("-#{ary[0]}") unless ary[0].nil?
|
492
|
+
set.add("--#{ary[1]}") unless ary[1].nil?
|
493
|
+
|
508
494
|
return set
|
509
495
|
end
|
510
|
-
|
496
|
+
|
511
497
|
def refresh_cmd(opts,args,cmd)
|
512
498
|
new_opts = {}
|
513
|
-
|
499
|
+
|
514
500
|
# Change symbols with dashes to underscores,
|
515
501
|
# so don't have to type @cmd_opts[:'dry-run'] all the time.
|
516
|
-
opts.each
|
517
|
-
|
518
|
-
key = key.gsub('-','_')
|
519
|
-
|
520
|
-
|
502
|
+
opts.each do |key,value|
|
503
|
+
# %s(max-retry) => :max_retry
|
504
|
+
key = key.to_s.gsub('-','_').to_sym
|
505
|
+
|
521
506
|
new_opts[key] = value
|
522
507
|
end
|
523
|
-
|
508
|
+
|
509
|
+
# For now don't set the default proc, as the original code
|
510
|
+
# did not have this in mind.
|
511
|
+
# Specifically, SiftCmd.build_sift_filename() is affected by
|
512
|
+
# this due to relying on @cmd_opts[:ext] to be nil.
|
513
|
+
# It's easy to change this one instance, but I'm not sure
|
514
|
+
# at the moment where else might be affected
|
515
|
+
## Cri has a default proc for default values
|
516
|
+
## that doesn't store the keys.
|
517
|
+
#new_opts.default_proc = proc do |hash,key|
|
518
|
+
# # :max_retry => %s(max-retry)
|
519
|
+
# key = key.to_s.gsub('_','-').to_sym
|
520
|
+
#
|
521
|
+
# opts.default_proc.call(hash,key)
|
522
|
+
#end
|
523
|
+
|
524
524
|
@cmd = cmd
|
525
525
|
@cmd_args = args
|
526
526
|
@cmd_opts = new_opts
|
527
|
-
|
527
|
+
|
528
528
|
return self
|
529
529
|
end
|
530
|
-
|
531
|
-
def run
|
530
|
+
|
531
|
+
def run
|
532
532
|
@app_cmd.run(@args)
|
533
533
|
end
|
534
|
-
|
535
|
-
def show_version
|
534
|
+
|
535
|
+
def show_version
|
536
536
|
puts "#{NAME} v#{VERSION}"
|
537
537
|
end
|
538
|
-
|
539
|
-
def sleep_scraper
|
538
|
+
|
539
|
+
def sleep_scraper
|
540
540
|
sleep(@sleep_time)
|
541
541
|
end
|
542
|
-
|
542
|
+
|
543
543
|
def start_spin(title,detail: '')
|
544
544
|
if @spinner.is_a?(Hash)
|
545
545
|
@spinner[:detail] = detail
|
546
546
|
@spinner[:title] = title
|
547
|
-
|
548
|
-
puts
|
547
|
+
|
548
|
+
puts(NO_SPINNER_MSG % @spinner)
|
549
549
|
else
|
550
550
|
@spinner.update(title: title,detail: detail)
|
551
|
-
@spinner.auto_spin
|
551
|
+
@spinner.auto_spin
|
552
552
|
end
|
553
553
|
end
|
554
|
-
|
555
|
-
def stop_spin
|
554
|
+
|
555
|
+
def stop_spin
|
556
556
|
if @spinner.is_a?(Hash)
|
557
557
|
puts (NO_SPINNER_MSG % @spinner) + ' done!'
|
558
558
|
else
|
559
|
-
@spinner.reset
|
559
|
+
@spinner.reset
|
560
560
|
@spinner.stop('done!')
|
561
561
|
end
|
562
562
|
end
|
563
|
-
|
563
|
+
|
564
564
|
def update_spin_detail(detail)
|
565
565
|
if @spinner.is_a?(Hash)
|
566
566
|
@spinner[:detail] = detail
|
567
|
-
|
568
|
-
puts
|
567
|
+
|
568
|
+
puts(NO_SPINNER_MSG % @spinner)
|
569
569
|
else
|
570
570
|
@spinner.tokens[:detail] = detail
|
571
571
|
end
|
572
572
|
end
|
573
573
|
end
|
574
|
-
|
574
|
+
|
575
575
|
###
|
576
|
-
# @author Jonathan Bradley Whited
|
576
|
+
# @author Jonathan Bradley Whited
|
577
577
|
# @since 0.2.0
|
578
578
|
###
|
579
579
|
class NoProgressBar
|
580
580
|
MSG = '%{title}... %{percent}%%'
|
581
581
|
PUT_INTERVAL = 100.0 / 6.25
|
582
582
|
MAX_PUT_INTERVAL = 100.0 + PUT_INTERVAL + 1.0
|
583
|
-
|
583
|
+
|
584
584
|
def initialize(title,total:,**tokens)
|
585
585
|
super()
|
586
|
-
|
586
|
+
|
587
587
|
@tokens = {title: title,total: total}
|
588
|
-
|
589
|
-
reset
|
590
|
-
|
588
|
+
|
589
|
+
reset
|
590
|
+
|
591
591
|
@tokens.merge!(tokens)
|
592
592
|
end
|
593
|
-
|
594
|
-
def reset
|
593
|
+
|
594
|
+
def reset
|
595
595
|
@tokens[:advance] = 0
|
596
596
|
@tokens[:percent] = 0
|
597
597
|
@tokens[:progress] = 0
|
598
598
|
end
|
599
|
-
|
599
|
+
|
600
600
|
def advance(progress=1)
|
601
601
|
total = @tokens[:total]
|
602
602
|
progress = @tokens[:progress] + progress
|
603
603
|
progress = total if progress > total
|
604
|
-
percent = (progress.to_f
|
605
|
-
|
604
|
+
percent = (progress.to_f / total.to_f * 100.0).round
|
605
|
+
|
606
606
|
@tokens[:percent] = percent
|
607
607
|
@tokens[:progress] = progress
|
608
|
-
|
608
|
+
|
609
609
|
if percent < 99.0
|
610
610
|
# Only output at certain intervals.
|
611
611
|
advance = @tokens[:advance]
|
612
612
|
i = 0.0
|
613
|
-
|
613
|
+
|
614
614
|
while i <= MAX_PUT_INTERVAL
|
615
615
|
if advance < i
|
616
616
|
break if percent >= i # Output
|
617
617
|
return # Don't output
|
618
618
|
end
|
619
|
-
|
619
|
+
|
620
620
|
i += PUT_INTERVAL
|
621
621
|
end
|
622
622
|
end
|
623
|
-
|
623
|
+
|
624
624
|
@tokens[:advance] = percent
|
625
|
-
|
626
|
-
puts to_s
|
625
|
+
|
626
|
+
puts to_s
|
627
627
|
end
|
628
|
-
|
629
|
-
def finish
|
628
|
+
|
629
|
+
def finish
|
630
630
|
advance(@tokens[:total])
|
631
631
|
end
|
632
|
-
|
633
|
-
def start
|
634
|
-
puts to_s
|
632
|
+
|
633
|
+
def start
|
634
|
+
puts to_s
|
635
635
|
end
|
636
|
-
|
637
|
-
def to_s
|
636
|
+
|
637
|
+
def to_s
|
638
638
|
return MSG % @tokens
|
639
639
|
end
|
640
640
|
end
|