unclekryon 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +6 -19
- data/Gemfile.lock +19 -23
- data/README.md +3 -2
- data/Rakefile +11 -30
- data/bin/unclekryon +4 -15
- data/hax/kryon.yaml +28 -0
- data/hax/kryon_aums_2002-2005.yaml +460 -0
- data/hax/kryon_aums_2006.yaml +601 -0
- data/hax/kryon_aums_2007.yaml +1024 -0
- data/hax/kryon_aums_2008.yaml +950 -0
- data/hax/kryon_aums_2009.yaml +496 -0
- data/hax/kryon_aums_2010.yaml +1443 -0
- data/hax/kryon_aums_2011.yaml +1458 -0
- data/hax/kryon_aums_2012.yaml +2123 -0
- data/hax/kryon_aums_2013.yaml +1647 -0
- data/hax/kryon_aums_2014.yaml +2478 -0
- data/hax/kryon_aums_2015.yaml +3386 -0
- data/hax/kryon_aums_2016.yaml +3476 -0
- data/hax/kryon_aums_2017.yaml +3712 -0
- data/hax/kryon_aums_2018.yaml +3654 -0
- data/lib/unclekryon.rb +165 -165
- data/lib/unclekryon/data/album_data.rb +74 -82
- data/lib/unclekryon/data/artist_data.rb +24 -36
- data/lib/unclekryon/data/artist_data_data.rb +29 -41
- data/lib/unclekryon/data/aum_data.rb +20 -32
- data/lib/unclekryon/data/base_data.rb +27 -39
- data/lib/unclekryon/data/pic_data.rb +25 -37
- data/lib/unclekryon/data/release_data.rb +14 -26
- data/lib/unclekryon/data/social_data.rb +6 -18
- data/lib/unclekryon/data/timespan_data.rb +16 -28
- data/lib/unclekryon/dev_opts.rb +7 -19
- data/lib/unclekryon/hacker.rb +119 -133
- data/lib/unclekryon/iso.rb +128 -138
- data/lib/unclekryon/iso/base_iso.rb +69 -81
- data/lib/unclekryon/iso/can_prov_terr.rb +34 -47
- data/lib/unclekryon/iso/country.rb +36 -49
- data/lib/unclekryon/iso/language.rb +86 -96
- data/lib/unclekryon/iso/region.rb +11 -25
- data/lib/unclekryon/iso/subregion.rb +11 -25
- data/lib/unclekryon/iso/usa_state.rb +28 -41
- data/lib/unclekryon/jsoner.rb +31 -50
- data/lib/unclekryon/log.rb +34 -46
- data/lib/unclekryon/parsers/kryon_aum_year_album_parser.rb +163 -167
- data/lib/unclekryon/parsers/kryon_aum_year_parser.rb +122 -127
- data/lib/unclekryon/server.rb +8 -17
- data/lib/unclekryon/trainer.rb +69 -83
- data/lib/unclekryon/uploader.rb +8 -17
- data/lib/unclekryon/util.rb +80 -92
- data/lib/unclekryon/version.rb +4 -16
- data/train/kryon.yaml +6077 -0
- data/unclekryon.gemspec +44 -42
- metadata +59 -16
data/lib/unclekryon.rb
CHANGED
@@ -1,28 +1,20 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of UncleKryon-server.
|
7
|
-
# Copyright (c) 2017-
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# UncleKryon-server is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with UncleKryon-server. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2017-2021 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
24
|
-
|
25
|
-
|
12
|
+
IS_SCRIPT = ($PROGRAM_NAME == __FILE__)
|
13
|
+
|
14
|
+
if IS_SCRIPT
|
15
|
+
require 'rubygems'
|
16
|
+
require 'bundler/setup'
|
17
|
+
end
|
26
18
|
|
27
19
|
require 'optparse'
|
28
20
|
|
@@ -60,10 +52,11 @@ require 'unclekryon/parsers/kryon_aum_year_parser'
|
|
60
52
|
|
61
53
|
module UncleKryon
|
62
54
|
class Main
|
63
|
-
OPT_HELP_UPDATED_ON =
|
64
|
-
|
55
|
+
OPT_HELP_UPDATED_ON = 'Change all "updated_*on" datetimes to <datetime>' \
|
56
|
+
" (e.g., #{Util.format_datetime(DateTime.now)})"
|
57
|
+
|
65
58
|
include Logging
|
66
|
-
|
59
|
+
|
67
60
|
def initialize(args)
|
68
61
|
@args = args
|
69
62
|
@cmd = nil
|
@@ -71,22 +64,22 @@ module UncleKryon
|
|
71
64
|
@options = {}
|
72
65
|
@parsers = []
|
73
66
|
end
|
74
|
-
|
75
|
-
def run
|
67
|
+
|
68
|
+
def run
|
76
69
|
parser = OptionParser.new do |op|
|
77
70
|
op.program_name = 'unclekryon'
|
78
71
|
op.version = VERSION
|
79
|
-
|
80
|
-
op.banner = <<~
|
72
|
+
|
73
|
+
op.banner = <<~BANNER
|
81
74
|
Usage: #{op.program_name} [options] <sub_cmd> [options] <sub_cmd>...
|
82
|
-
|
75
|
+
|
83
76
|
Sub Commands:
|
84
77
|
hax kryon aum year
|
85
78
|
iso list
|
86
|
-
|
79
|
+
|
87
80
|
Options:
|
88
|
-
|
89
|
-
|
81
|
+
BANNER
|
82
|
+
|
90
83
|
op.on('-d','--dev','Raise errors on missing data, etc.') do
|
91
84
|
DevOpts.instance.dev = true
|
92
85
|
end
|
@@ -97,129 +90,150 @@ module UncleKryon
|
|
97
90
|
op.on('-t','--test','Fill in training data with random values, etc. for fast testing') do
|
98
91
|
DevOpts.instance.test = true
|
99
92
|
end
|
100
|
-
op.on('-v','--version','Print version to console')
|
93
|
+
op.on('-v','--version','Print version to console') do
|
94
|
+
puts "#{op.program_name} v#{op.version}"
|
95
|
+
exit
|
96
|
+
end
|
101
97
|
end
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
parse_iso_cmd()
|
98
|
+
|
99
|
+
add_parser(parser)
|
100
|
+
|
101
|
+
if shift_args
|
102
|
+
parse_hax_cmd
|
103
|
+
parse_iso_cmd
|
109
104
|
end
|
110
|
-
|
111
|
-
if
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
puts p
|
117
|
-
puts
|
118
|
-
end
|
119
|
-
|
120
|
-
s = <<~EOS
|
121
|
-
Examples:
|
122
|
-
| # To view all of the options for the sub commands:
|
123
|
-
| $ #{parser.program_name} hax kryon aum year
|
124
|
-
| $ #{parser.program_name} iso list
|
125
|
-
|
|
126
|
-
| <hax>:
|
127
|
-
| # Train the data 1st before haxing (if there is no training data)
|
128
|
-
| $ #{parser.program_name} -d hax -t kryon aum year -t 2017 -s
|
129
|
-
| $ #{parser.program_name} -d hax -t kryon aum year -t 2017 -a 2.2
|
130
|
-
|
|
131
|
-
| # Hax the data (even though --title is not required, it is recommended)
|
132
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -s
|
133
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -a 10.9
|
134
|
-
| $ #{parser.program_name} -d hax kryon aum year -a 2017.9.29
|
135
|
-
|
|
136
|
-
| # Hax the 2nd "6.4" album (if there are 2)
|
137
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -a 6.4:2
|
138
|
-
|
|
139
|
-
| <iso>:
|
140
|
-
| $ #{parser.program_name} -n iso -u '2019-04-16 11:11:00'
|
141
|
-
| $ #{parser.program_name} iso list -r
|
142
|
-
EOS
|
143
|
-
puts s.gsub(/^\|/,'')
|
144
|
-
end
|
105
|
+
|
106
|
+
return if @did_cmd
|
107
|
+
|
108
|
+
@parsers.each do |p|
|
109
|
+
puts p
|
110
|
+
puts
|
145
111
|
end
|
112
|
+
|
113
|
+
puts(<<~EXAMPLES)
|
114
|
+
Examples:
|
115
|
+
# To view all of the options for the sub commands:
|
116
|
+
$ #{parser.program_name} hax kryon aum year
|
117
|
+
$ #{parser.program_name} iso list
|
118
|
+
|
119
|
+
<hax>:
|
120
|
+
# Train the data 1st before haxing (if there is no training data)
|
121
|
+
$ #{parser.program_name} -d hax -t kryon aum year -t 2017 -s
|
122
|
+
$ #{parser.program_name} -d hax -t kryon aum year -t 2017 -a 2.2
|
123
|
+
|
124
|
+
# Hax the data (even though --title is not required, it is recommended)
|
125
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -s
|
126
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -a 10.9
|
127
|
+
$ #{parser.program_name} -d hax kryon aum year -a 2017.9.29
|
128
|
+
|
129
|
+
# Hax the 2nd "6.4" album (if there are 2)
|
130
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -a 6.4:2
|
131
|
+
|
132
|
+
<iso>:
|
133
|
+
$ #{parser.program_name} -n iso -u '2019-04-16 11:11:00'
|
134
|
+
$ #{parser.program_name} iso list -r
|
135
|
+
EXAMPLES
|
146
136
|
end
|
147
|
-
|
148
|
-
def
|
149
|
-
|
137
|
+
|
138
|
+
def add_parser(parser)
|
139
|
+
@parsers.push(parser)
|
140
|
+
parser.order!(@args,into: @options)
|
141
|
+
end
|
142
|
+
|
143
|
+
def shift_args
|
144
|
+
return !@args.nil? && !(@cmd = @args.shift).nil?
|
145
|
+
end
|
146
|
+
|
147
|
+
def log_opts
|
148
|
+
log.info("Using options#{@options}")
|
149
|
+
end
|
150
|
+
|
151
|
+
def cmd?(cmd)
|
152
|
+
return false if @did_cmd || @cmd.nil?
|
153
|
+
|
154
|
+
user_cmd = @cmd.gsub(/[[:space:]]+/,'').downcase
|
155
|
+
cmd = cmd.gsub(/[[:space:]]+/,'').downcase
|
156
|
+
|
157
|
+
return cmd.start_with?(user_cmd)
|
158
|
+
end
|
159
|
+
|
160
|
+
def do_cmd?
|
161
|
+
return !@options[:help] && !@options[:version]
|
150
162
|
end
|
151
|
-
|
152
|
-
def parse_hax_cmd
|
163
|
+
|
164
|
+
def parse_hax_cmd
|
153
165
|
return if !cmd?('hax')
|
154
|
-
|
166
|
+
|
155
167
|
parser = OptionParser.new do |op|
|
156
168
|
op.banner = '<hax> options:'
|
157
169
|
|
158
|
-
op.on('-d','--dir <dir>',
|
170
|
+
op.on('-d','--dir <dir>','Directory to save the hax data to' \
|
171
|
+
" (default: #{Hacker::HAX_DIRNAME})") do |dir|
|
159
172
|
@options[:hax_dirname] = dir
|
160
173
|
end
|
161
174
|
op.on('-t','--train','Train the data using machine learning')
|
162
|
-
op.on('-i','--train-dir <dir>',
|
175
|
+
op.on('-i','--train-dir <dir>','Directory to save the training data to' \
|
176
|
+
" (default: #{Hacker::TRAIN_DIRNAME})") do |dir|
|
163
177
|
@options[:train_dirname] = dir
|
164
178
|
end
|
165
179
|
op.on('-u','--updated-on <datetime>',OPT_HELP_UPDATED_ON) do |datetime|
|
166
180
|
@options[:updated_on] = datetime
|
167
181
|
end
|
168
182
|
end
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
if do_cmd?()
|
183
|
+
|
184
|
+
add_parser(parser)
|
185
|
+
|
186
|
+
if do_cmd?
|
174
187
|
if @options[:updated_on]
|
175
|
-
hax_dirname = @options[:hax_dirname]
|
188
|
+
hax_dirname = @options[:hax_dirname] || Hacker::HAX_DIRNAME
|
176
189
|
gsub_updated_on(hax_dirname,@options[:updated_on])
|
177
|
-
|
190
|
+
|
178
191
|
@did_cmd = true
|
179
192
|
end
|
180
193
|
end
|
181
|
-
|
182
|
-
if shift_args
|
183
|
-
parse_hax_kryon_cmd
|
194
|
+
|
195
|
+
if shift_args
|
196
|
+
parse_hax_kryon_cmd
|
184
197
|
end
|
185
198
|
end
|
186
|
-
|
187
|
-
def parse_hax_kryon_cmd
|
199
|
+
|
200
|
+
def parse_hax_kryon_cmd
|
188
201
|
return if !cmd?('kryon')
|
189
|
-
|
202
|
+
|
190
203
|
parser = OptionParser.new do |op|
|
191
204
|
op.banner = '<kryon> options:'
|
192
|
-
|
193
|
-
op.on('-f','--file <file>',
|
205
|
+
|
206
|
+
op.on('-f','--file <file>','File to save the hax data to' \
|
207
|
+
" (default: #{Hacker::HAX_KRYON_FILENAME})") do |file|
|
194
208
|
@options[:hax_kryon_filename] = file
|
195
209
|
end
|
196
|
-
op.on('-t','--train-file <file>',
|
210
|
+
op.on('-t','--train-file <file>','File to save the training data to' \
|
211
|
+
" (default: #{Hacker::TRAIN_KRYON_FILENAME})") do |file|
|
197
212
|
@options[:train_kryon_filename] = file
|
198
213
|
end
|
199
214
|
end
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
parse_hax_kryon_aum_cmd()
|
215
|
+
|
216
|
+
add_parser(parser)
|
217
|
+
|
218
|
+
if shift_args
|
219
|
+
parse_hax_kryon_aum_cmd
|
206
220
|
end
|
207
221
|
end
|
208
|
-
|
209
|
-
def parse_hax_kryon_aum_cmd
|
222
|
+
|
223
|
+
def parse_hax_kryon_aum_cmd
|
210
224
|
return if !cmd?('aum')
|
211
|
-
|
212
|
-
if shift_args
|
213
|
-
parse_hax_kryon_aum_year_cmd
|
225
|
+
|
226
|
+
if shift_args
|
227
|
+
parse_hax_kryon_aum_year_cmd
|
214
228
|
end
|
215
229
|
end
|
216
|
-
|
217
|
-
def parse_hax_kryon_aum_year_cmd
|
230
|
+
|
231
|
+
def parse_hax_kryon_aum_year_cmd
|
218
232
|
return if !cmd?('year')
|
219
|
-
|
233
|
+
|
220
234
|
parser = OptionParser.new do |op|
|
221
235
|
op.banner = '<year> options:'
|
222
|
-
|
236
|
+
|
223
237
|
op.on('-t','--title <title>','Title of year release to hack (e.g., 2017)')
|
224
238
|
op.on('-a','--album <album>','Album to hack (e.g., 2017.12.25, 1.10, 6.4:2)')
|
225
239
|
op.on('-s','--albums','Hack all albums')
|
@@ -228,15 +242,14 @@ module UncleKryon
|
|
228
242
|
@options[:begin_album] = begin_album
|
229
243
|
end
|
230
244
|
end
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
245
|
+
|
246
|
+
add_parser(parser)
|
247
|
+
|
248
|
+
if do_cmd?
|
249
|
+
log_opts
|
250
|
+
|
251
|
+
hacker = Hacker.new(**@options)
|
252
|
+
|
240
253
|
if @options[:train]
|
241
254
|
if @options[:album]
|
242
255
|
hacker.train_kryon_aum_year_album(@options[:album],@options[:title])
|
@@ -247,7 +260,7 @@ module UncleKryon
|
|
247
260
|
else
|
248
261
|
hacker.train_kryon_aum_year(@options[:title])
|
249
262
|
end
|
250
|
-
|
263
|
+
|
251
264
|
@did_cmd = true
|
252
265
|
end
|
253
266
|
else
|
@@ -260,50 +273,50 @@ module UncleKryon
|
|
260
273
|
else
|
261
274
|
hacker.parse_kryon_aum_year(@options[:title])
|
262
275
|
end
|
263
|
-
|
276
|
+
|
264
277
|
@did_cmd = true
|
265
278
|
end
|
266
279
|
end
|
267
280
|
end
|
268
281
|
end
|
269
|
-
|
270
|
-
def parse_iso_cmd
|
282
|
+
|
283
|
+
def parse_iso_cmd
|
271
284
|
return if !cmd?('iso')
|
272
|
-
|
285
|
+
|
273
286
|
parser = OptionParser.new do |op|
|
274
287
|
op.banner = '<iso> options:'
|
275
|
-
|
276
|
-
op.on('-d','--dir <dir>',
|
288
|
+
|
289
|
+
op.on('-d','--dir <dir>','Directory to read/write ISO data' \
|
290
|
+
" (default: #{BaseIsos::DEFAULT_DIR})") do |dir|
|
277
291
|
@options[:iso_dirname] = dir
|
278
292
|
end
|
279
293
|
op.on('-u','--updated-on <datetime>',OPT_HELP_UPDATED_ON) do |datetime|
|
280
294
|
@options[:updated_on] = datetime
|
281
295
|
end
|
282
296
|
end
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
if do_cmd?()
|
297
|
+
|
298
|
+
add_parser(parser)
|
299
|
+
|
300
|
+
if do_cmd?
|
288
301
|
if @options[:updated_on]
|
289
|
-
iso_dirname = @options[:iso_dirname]
|
302
|
+
iso_dirname = @options[:iso_dirname] || BaseIsos::DEFAULT_DIR
|
290
303
|
gsub_updated_on(iso_dirname,@options[:updated_on])
|
291
|
-
|
304
|
+
|
292
305
|
@did_cmd = true
|
293
306
|
end
|
294
307
|
end
|
295
|
-
|
296
|
-
if shift_args
|
297
|
-
parse_iso_list_cmd
|
308
|
+
|
309
|
+
if shift_args
|
310
|
+
parse_iso_list_cmd
|
298
311
|
end
|
299
312
|
end
|
300
|
-
|
301
|
-
def parse_iso_list_cmd
|
313
|
+
|
314
|
+
def parse_iso_list_cmd
|
302
315
|
return if !cmd?('list')
|
303
|
-
|
316
|
+
|
304
317
|
parser = OptionParser.new do |op|
|
305
318
|
op.banner = '<list> options:'
|
306
|
-
|
319
|
+
|
307
320
|
op.on('-c','--canada','List Canadian provinces and territories')
|
308
321
|
op.on('-u','--usa','List USA states')
|
309
322
|
op.on('-o','--country','List countries')
|
@@ -311,11 +324,10 @@ module UncleKryon
|
|
311
324
|
op.on('-r','--region','List regions (i.e., continents, etc.)')
|
312
325
|
op.on('-s','--subregion','List subregions')
|
313
326
|
end
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
if do_cmd?()
|
327
|
+
|
328
|
+
add_parser(parser)
|
329
|
+
|
330
|
+
if do_cmd?
|
319
331
|
if @options[:canada]
|
320
332
|
puts Iso.can_provs_terrs
|
321
333
|
@did_cmd = true
|
@@ -337,15 +349,15 @@ module UncleKryon
|
|
337
349
|
end
|
338
350
|
end
|
339
351
|
end
|
340
|
-
|
352
|
+
|
341
353
|
def gsub_updated_on(dirname,updated_on)
|
342
354
|
updated_on = Util.parse_datetime_s(updated_on) # Raise errors on bad format
|
343
355
|
updated_on = Util.format_datetime(updated_on)
|
344
|
-
|
356
|
+
|
345
357
|
Dir.glob(File.join(dirname,'*.yaml')) do |filepath|
|
346
358
|
lines = IO.readlines(filepath)
|
347
359
|
update_count = 0
|
348
|
-
|
360
|
+
|
349
361
|
lines.each_with_index do |line,i|
|
350
362
|
if line =~ /\A\s*updated\_.*on\:.*\Z/i
|
351
363
|
line = line.split(':')[0] << ": '#{updated_on}'"
|
@@ -353,32 +365,20 @@ module UncleKryon
|
|
353
365
|
update_count += 1
|
354
366
|
end
|
355
367
|
end
|
356
|
-
|
368
|
+
|
357
369
|
if !@options[:no_clobber]
|
358
370
|
File.open(filepath,'w') do |file|
|
359
371
|
file.puts lines
|
360
372
|
end
|
361
373
|
end
|
362
|
-
|
363
|
-
puts %Q
|
374
|
+
|
375
|
+
puts %Q("#{filepath}" updated_on: #{update_count})
|
364
376
|
end
|
365
377
|
end
|
366
|
-
|
367
|
-
def log_opts()
|
368
|
-
log.info("Using options#{@options}")
|
369
|
-
end
|
370
|
-
|
371
|
-
def cmd?(cmd)
|
372
|
-
return !@did_cmd && !@cmd.nil?() && @cmd.match?(/\A[[:space:]]*#{Regexp.escape(cmd)}[[:space:]]*\z/i)
|
373
|
-
end
|
374
|
-
|
375
|
-
def do_cmd?()
|
376
|
-
return !@options[:help] && !@options[:version]
|
377
|
-
end
|
378
378
|
end
|
379
379
|
end
|
380
380
|
|
381
|
-
if
|
381
|
+
if IS_SCRIPT
|
382
382
|
main = UncleKryon::Main.new(ARGV)
|
383
|
-
main.run
|
383
|
+
main.run
|
384
384
|
end
|