unclekryon 0.4.9.pre.alpha → 0.4.12.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +10 -26
- data/Gemfile.lock +20 -24
- data/README.md +3 -2
- data/Rakefile +11 -30
- data/bin/unclekryon +4 -15
- data/hax/kryon.yaml +28 -0
- data/hax/kryon_aums_2002-2005.yaml +460 -0
- data/hax/kryon_aums_2006.yaml +601 -0
- data/hax/kryon_aums_2007.yaml +1024 -0
- data/hax/kryon_aums_2008.yaml +950 -0
- data/hax/kryon_aums_2009.yaml +496 -0
- data/hax/kryon_aums_2010.yaml +1443 -0
- data/hax/kryon_aums_2011.yaml +1458 -0
- data/hax/kryon_aums_2012.yaml +2123 -0
- data/hax/kryon_aums_2013.yaml +1647 -0
- data/hax/kryon_aums_2014.yaml +2478 -0
- data/hax/kryon_aums_2015.yaml +3386 -0
- data/hax/kryon_aums_2016.yaml +3476 -0
- data/hax/kryon_aums_2017.yaml +3712 -0
- data/hax/kryon_aums_2018.yaml +3654 -0
- data/lib/unclekryon/data/album_data.rb +74 -82
- data/lib/unclekryon/data/artist_data.rb +24 -36
- data/lib/unclekryon/data/artist_data_data.rb +29 -41
- data/lib/unclekryon/data/aum_data.rb +20 -32
- data/lib/unclekryon/data/base_data.rb +27 -39
- data/lib/unclekryon/data/pic_data.rb +25 -37
- data/lib/unclekryon/data/release_data.rb +14 -26
- data/lib/unclekryon/data/social_data.rb +6 -18
- data/lib/unclekryon/data/timespan_data.rb +16 -28
- data/lib/unclekryon/dev_opts.rb +7 -19
- data/lib/unclekryon/hacker.rb +121 -135
- data/lib/unclekryon/iso/base_iso.rb +69 -81
- data/lib/unclekryon/iso/can_prov_terr.rb +34 -47
- data/lib/unclekryon/iso/country.rb +34 -51
- data/lib/unclekryon/iso/language.rb +84 -98
- data/lib/unclekryon/iso/region.rb +8 -29
- data/lib/unclekryon/iso/subregion.rb +8 -29
- data/lib/unclekryon/iso/usa_state.rb +28 -41
- data/lib/unclekryon/iso.rb +128 -138
- data/lib/unclekryon/jsoner.rb +31 -50
- data/lib/unclekryon/log.rb +34 -46
- data/lib/unclekryon/parsers/kryon_aum_year_album_parser.rb +163 -167
- data/lib/unclekryon/parsers/kryon_aum_year_parser.rb +122 -127
- data/lib/unclekryon/server.rb +8 -17
- data/lib/unclekryon/trainer.rb +68 -85
- data/lib/unclekryon/uploader.rb +8 -17
- data/lib/unclekryon/util.rb +80 -92
- data/lib/unclekryon/version.rb +4 -16
- data/lib/unclekryon.rb +166 -166
- data/train/kryon.yaml +6077 -0
- data/unclekryon.gemspec +49 -49
- metadata +50 -22
data/lib/unclekryon.rb
CHANGED
@@ -1,28 +1,20 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of UncleKryon-server.
|
7
|
-
# Copyright (c) 2017-
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# UncleKryon-server is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with UncleKryon-server. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2017-2022 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
24
|
-
|
25
|
-
|
12
|
+
IS_SCRIPT = ($PROGRAM_NAME == __FILE__)
|
13
|
+
|
14
|
+
if IS_SCRIPT
|
15
|
+
require 'rubygems'
|
16
|
+
require 'bundler/setup'
|
17
|
+
end
|
26
18
|
|
27
19
|
require 'optparse'
|
28
20
|
|
@@ -60,10 +52,11 @@ require 'unclekryon/parsers/kryon_aum_year_parser'
|
|
60
52
|
|
61
53
|
module UncleKryon
|
62
54
|
class Main
|
63
|
-
OPT_HELP_UPDATED_ON =
|
64
|
-
|
55
|
+
OPT_HELP_UPDATED_ON = 'Change all "updated_*on" datetimes to <datetime>' \
|
56
|
+
" (e.g., #{Util.format_datetime(DateTime.now)})"
|
57
|
+
|
65
58
|
include Logging
|
66
|
-
|
59
|
+
|
67
60
|
def initialize(args)
|
68
61
|
@args = args
|
69
62
|
@cmd = nil
|
@@ -71,22 +64,22 @@ module UncleKryon
|
|
71
64
|
@options = {}
|
72
65
|
@parsers = []
|
73
66
|
end
|
74
|
-
|
75
|
-
def run
|
67
|
+
|
68
|
+
def run
|
76
69
|
parser = OptionParser.new do |op|
|
77
70
|
op.program_name = 'unclekryon'
|
78
71
|
op.version = VERSION
|
79
|
-
|
80
|
-
op.banner = <<~
|
72
|
+
|
73
|
+
op.banner = <<~BANNER
|
81
74
|
Usage: #{op.program_name} [options] <sub_cmd> [options] <sub_cmd>...
|
82
|
-
|
75
|
+
|
83
76
|
Sub Commands:
|
84
77
|
hax kryon aum year
|
85
78
|
iso list
|
86
|
-
|
79
|
+
|
87
80
|
Options:
|
88
|
-
|
89
|
-
|
81
|
+
BANNER
|
82
|
+
|
90
83
|
op.on('-d','--dev','Raise errors on missing data, etc.') do
|
91
84
|
DevOpts.instance.dev = true
|
92
85
|
end
|
@@ -97,129 +90,150 @@ module UncleKryon
|
|
97
90
|
op.on('-t','--test','Fill in training data with random values, etc. for fast testing') do
|
98
91
|
DevOpts.instance.test = true
|
99
92
|
end
|
100
|
-
op.on('-v','--version','Print version to console')
|
93
|
+
op.on('-v','--version','Print version to console') do
|
94
|
+
puts "#{op.program_name} v#{op.version}"
|
95
|
+
exit
|
96
|
+
end
|
101
97
|
end
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
parse_iso_cmd()
|
98
|
+
|
99
|
+
add_parser(parser)
|
100
|
+
|
101
|
+
if shift_args
|
102
|
+
parse_hax_cmd
|
103
|
+
parse_iso_cmd
|
109
104
|
end
|
110
|
-
|
111
|
-
if
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
puts p
|
117
|
-
puts
|
118
|
-
end
|
119
|
-
|
120
|
-
s = <<~EOS
|
121
|
-
Examples:
|
122
|
-
| # To view all of the options for the sub commands:
|
123
|
-
| $ #{parser.program_name} hax kryon aum year
|
124
|
-
| $ #{parser.program_name} iso list
|
125
|
-
|
|
126
|
-
| <hax>:
|
127
|
-
| # Train the data 1st before haxing (if there is no training data)
|
128
|
-
| $ #{parser.program_name} -d hax -t kryon aum year -t 2017 -s
|
129
|
-
| $ #{parser.program_name} -d hax -t kryon aum year -t 2017 -a 2.2
|
130
|
-
|
|
131
|
-
| # Hax the data (even though --title is not required, it is recommended)
|
132
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -s
|
133
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -a 10.9
|
134
|
-
| $ #{parser.program_name} -d hax kryon aum year -a 2017.9.29
|
135
|
-
|
|
136
|
-
| # Hax the 2nd "6.4" album (if there are 2)
|
137
|
-
| $ #{parser.program_name} -d hax kryon aum year -t 2017 -a 6.4:2
|
138
|
-
|
|
139
|
-
| <iso>:
|
140
|
-
| $ #{parser.program_name} -n iso -u '2019-04-16 11:11:00'
|
141
|
-
| $ #{parser.program_name} iso list -r
|
142
|
-
EOS
|
143
|
-
puts s.gsub(/^\|/,'')
|
144
|
-
end
|
105
|
+
|
106
|
+
return if @did_cmd
|
107
|
+
|
108
|
+
@parsers.each do |p|
|
109
|
+
puts p
|
110
|
+
puts
|
145
111
|
end
|
112
|
+
|
113
|
+
puts(<<~EXAMPLES)
|
114
|
+
Examples:
|
115
|
+
# To view all of the options for the sub commands:
|
116
|
+
$ #{parser.program_name} hax kryon aum year
|
117
|
+
$ #{parser.program_name} iso list
|
118
|
+
|
119
|
+
<hax>:
|
120
|
+
# Train the data 1st before haxing (if there is no training data)
|
121
|
+
$ #{parser.program_name} -d hax -t kryon aum year -t 2017 -s
|
122
|
+
$ #{parser.program_name} -d hax -t kryon aum year -t 2017 -a 2.2
|
123
|
+
|
124
|
+
# Hax the data (even though --title is not required, it is recommended)
|
125
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -s
|
126
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -a 10.9
|
127
|
+
$ #{parser.program_name} -d hax kryon aum year -a 2017.9.29
|
128
|
+
|
129
|
+
# Hax the 2nd "6.4" album (if there are 2)
|
130
|
+
$ #{parser.program_name} -d hax kryon aum year -t 2017 -a 6.4:2
|
131
|
+
|
132
|
+
<iso>:
|
133
|
+
$ #{parser.program_name} -n iso -u '2019-04-16 11:11:00'
|
134
|
+
$ #{parser.program_name} iso list -r
|
135
|
+
EXAMPLES
|
146
136
|
end
|
147
|
-
|
148
|
-
def
|
149
|
-
|
137
|
+
|
138
|
+
def add_parser(parser)
|
139
|
+
@parsers.push(parser)
|
140
|
+
parser.order!(@args,into: @options)
|
141
|
+
end
|
142
|
+
|
143
|
+
def shift_args
|
144
|
+
return !@args.nil? && !(@cmd = @args.shift).nil?
|
145
|
+
end
|
146
|
+
|
147
|
+
def log_opts
|
148
|
+
log.info("Using options#{@options}")
|
149
|
+
end
|
150
|
+
|
151
|
+
def cmd?(cmd)
|
152
|
+
return false if @did_cmd || @cmd.nil?
|
153
|
+
|
154
|
+
user_cmd = @cmd.gsub(/[[:space:]]+/,'').downcase
|
155
|
+
cmd = cmd.gsub(/[[:space:]]+/,'').downcase
|
156
|
+
|
157
|
+
return cmd.start_with?(user_cmd)
|
158
|
+
end
|
159
|
+
|
160
|
+
def do_cmd?
|
161
|
+
return !@options[:help] && !@options[:version]
|
150
162
|
end
|
151
|
-
|
152
|
-
def parse_hax_cmd
|
163
|
+
|
164
|
+
def parse_hax_cmd
|
153
165
|
return if !cmd?('hax')
|
154
|
-
|
166
|
+
|
155
167
|
parser = OptionParser.new do |op|
|
156
168
|
op.banner = '<hax> options:'
|
157
169
|
|
158
|
-
op.on('-d','--dir <dir>',
|
170
|
+
op.on('-d','--dir <dir>','Directory to save the hax data to' \
|
171
|
+
" (default: #{Hacker::HAX_DIRNAME})") do |dir|
|
159
172
|
@options[:hax_dirname] = dir
|
160
173
|
end
|
161
174
|
op.on('-t','--train','Train the data using machine learning')
|
162
|
-
op.on('-i','--train-dir <dir>',
|
175
|
+
op.on('-i','--train-dir <dir>','Directory to save the training data to' \
|
176
|
+
" (default: #{Hacker::TRAIN_DIRNAME})") do |dir|
|
163
177
|
@options[:train_dirname] = dir
|
164
178
|
end
|
165
179
|
op.on('-u','--updated-on <datetime>',OPT_HELP_UPDATED_ON) do |datetime|
|
166
180
|
@options[:updated_on] = datetime
|
167
181
|
end
|
168
182
|
end
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
if do_cmd?()
|
183
|
+
|
184
|
+
add_parser(parser)
|
185
|
+
|
186
|
+
if do_cmd?
|
174
187
|
if @options[:updated_on]
|
175
|
-
hax_dirname = @options[:hax_dirname]
|
188
|
+
hax_dirname = @options[:hax_dirname] || Hacker::HAX_DIRNAME
|
176
189
|
gsub_updated_on(hax_dirname,@options[:updated_on])
|
177
|
-
|
190
|
+
|
178
191
|
@did_cmd = true
|
179
192
|
end
|
180
193
|
end
|
181
|
-
|
182
|
-
if shift_args
|
183
|
-
parse_hax_kryon_cmd
|
194
|
+
|
195
|
+
if shift_args
|
196
|
+
parse_hax_kryon_cmd
|
184
197
|
end
|
185
198
|
end
|
186
|
-
|
187
|
-
def parse_hax_kryon_cmd
|
199
|
+
|
200
|
+
def parse_hax_kryon_cmd
|
188
201
|
return if !cmd?('kryon')
|
189
|
-
|
202
|
+
|
190
203
|
parser = OptionParser.new do |op|
|
191
204
|
op.banner = '<kryon> options:'
|
192
|
-
|
193
|
-
op.on('-f','--file <file>',
|
205
|
+
|
206
|
+
op.on('-f','--file <file>','File to save the hax data to' \
|
207
|
+
" (default: #{Hacker::HAX_KRYON_FILENAME})") do |file|
|
194
208
|
@options[:hax_kryon_filename] = file
|
195
209
|
end
|
196
|
-
op.on('-t','--train-file <file>',
|
210
|
+
op.on('-t','--train-file <file>','File to save the training data to' \
|
211
|
+
" (default: #{Hacker::TRAIN_KRYON_FILENAME})") do |file|
|
197
212
|
@options[:train_kryon_filename] = file
|
198
213
|
end
|
199
214
|
end
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
parse_hax_kryon_aum_cmd()
|
215
|
+
|
216
|
+
add_parser(parser)
|
217
|
+
|
218
|
+
if shift_args
|
219
|
+
parse_hax_kryon_aum_cmd
|
206
220
|
end
|
207
221
|
end
|
208
|
-
|
209
|
-
def parse_hax_kryon_aum_cmd
|
222
|
+
|
223
|
+
def parse_hax_kryon_aum_cmd
|
210
224
|
return if !cmd?('aum')
|
211
|
-
|
212
|
-
if shift_args
|
213
|
-
parse_hax_kryon_aum_year_cmd
|
225
|
+
|
226
|
+
if shift_args
|
227
|
+
parse_hax_kryon_aum_year_cmd
|
214
228
|
end
|
215
229
|
end
|
216
|
-
|
217
|
-
def parse_hax_kryon_aum_year_cmd
|
230
|
+
|
231
|
+
def parse_hax_kryon_aum_year_cmd
|
218
232
|
return if !cmd?('year')
|
219
|
-
|
233
|
+
|
220
234
|
parser = OptionParser.new do |op|
|
221
235
|
op.banner = '<year> options:'
|
222
|
-
|
236
|
+
|
223
237
|
op.on('-t','--title <title>','Title of year release to hack (e.g., 2017)')
|
224
238
|
op.on('-a','--album <album>','Album to hack (e.g., 2017.12.25, 1.10, 6.4:2)')
|
225
239
|
op.on('-s','--albums','Hack all albums')
|
@@ -228,15 +242,14 @@ module UncleKryon
|
|
228
242
|
@options[:begin_album] = begin_album
|
229
243
|
end
|
230
244
|
end
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
245
|
+
|
246
|
+
add_parser(parser)
|
247
|
+
|
248
|
+
if do_cmd?
|
249
|
+
log_opts
|
250
|
+
|
251
|
+
hacker = Hacker.new(**@options)
|
252
|
+
|
240
253
|
if @options[:train]
|
241
254
|
if @options[:album]
|
242
255
|
hacker.train_kryon_aum_year_album(@options[:album],@options[:title])
|
@@ -247,7 +260,7 @@ module UncleKryon
|
|
247
260
|
else
|
248
261
|
hacker.train_kryon_aum_year(@options[:title])
|
249
262
|
end
|
250
|
-
|
263
|
+
|
251
264
|
@did_cmd = true
|
252
265
|
end
|
253
266
|
else
|
@@ -260,50 +273,50 @@ module UncleKryon
|
|
260
273
|
else
|
261
274
|
hacker.parse_kryon_aum_year(@options[:title])
|
262
275
|
end
|
263
|
-
|
276
|
+
|
264
277
|
@did_cmd = true
|
265
278
|
end
|
266
279
|
end
|
267
280
|
end
|
268
281
|
end
|
269
|
-
|
270
|
-
def parse_iso_cmd
|
282
|
+
|
283
|
+
def parse_iso_cmd
|
271
284
|
return if !cmd?('iso')
|
272
|
-
|
285
|
+
|
273
286
|
parser = OptionParser.new do |op|
|
274
287
|
op.banner = '<iso> options:'
|
275
|
-
|
276
|
-
op.on('-d','--dir <dir>',
|
288
|
+
|
289
|
+
op.on('-d','--dir <dir>','Directory to read/write ISO data' \
|
290
|
+
" (default: #{BaseIsos::DEFAULT_DIR})") do |dir|
|
277
291
|
@options[:iso_dirname] = dir
|
278
292
|
end
|
279
293
|
op.on('-u','--updated-on <datetime>',OPT_HELP_UPDATED_ON) do |datetime|
|
280
294
|
@options[:updated_on] = datetime
|
281
295
|
end
|
282
296
|
end
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
if do_cmd?()
|
297
|
+
|
298
|
+
add_parser(parser)
|
299
|
+
|
300
|
+
if do_cmd?
|
288
301
|
if @options[:updated_on]
|
289
|
-
iso_dirname = @options[:iso_dirname]
|
302
|
+
iso_dirname = @options[:iso_dirname] || BaseIsos::DEFAULT_DIR
|
290
303
|
gsub_updated_on(iso_dirname,@options[:updated_on])
|
291
|
-
|
304
|
+
|
292
305
|
@did_cmd = true
|
293
306
|
end
|
294
307
|
end
|
295
|
-
|
296
|
-
if shift_args
|
297
|
-
parse_iso_list_cmd
|
308
|
+
|
309
|
+
if shift_args
|
310
|
+
parse_iso_list_cmd
|
298
311
|
end
|
299
312
|
end
|
300
|
-
|
301
|
-
def parse_iso_list_cmd
|
313
|
+
|
314
|
+
def parse_iso_list_cmd
|
302
315
|
return if !cmd?('list')
|
303
|
-
|
316
|
+
|
304
317
|
parser = OptionParser.new do |op|
|
305
318
|
op.banner = '<list> options:'
|
306
|
-
|
319
|
+
|
307
320
|
op.on('-c','--canada','List Canadian provinces and territories')
|
308
321
|
op.on('-u','--usa','List USA states')
|
309
322
|
op.on('-o','--country','List countries')
|
@@ -311,11 +324,10 @@ module UncleKryon
|
|
311
324
|
op.on('-r','--region','List regions (i.e., continents, etc.)')
|
312
325
|
op.on('-s','--subregion','List subregions')
|
313
326
|
end
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
if do_cmd?()
|
327
|
+
|
328
|
+
add_parser(parser)
|
329
|
+
|
330
|
+
if do_cmd?
|
319
331
|
if @options[:canada]
|
320
332
|
puts Iso.can_provs_terrs
|
321
333
|
@did_cmd = true
|
@@ -337,15 +349,15 @@ module UncleKryon
|
|
337
349
|
end
|
338
350
|
end
|
339
351
|
end
|
340
|
-
|
352
|
+
|
341
353
|
def gsub_updated_on(dirname,updated_on)
|
342
354
|
updated_on = Util.parse_datetime_s(updated_on) # Raise errors on bad format
|
343
355
|
updated_on = Util.format_datetime(updated_on)
|
344
|
-
|
356
|
+
|
345
357
|
Dir.glob(File.join(dirname,'*.yaml')) do |filepath|
|
346
|
-
lines =
|
358
|
+
lines = File.readlines(filepath)
|
347
359
|
update_count = 0
|
348
|
-
|
360
|
+
|
349
361
|
lines.each_with_index do |line,i|
|
350
362
|
if line =~ /\A\s*updated\_.*on\:.*\Z/i
|
351
363
|
line = line.split(':')[0] << ": '#{updated_on}'"
|
@@ -353,32 +365,20 @@ module UncleKryon
|
|
353
365
|
update_count += 1
|
354
366
|
end
|
355
367
|
end
|
356
|
-
|
368
|
+
|
357
369
|
if !@options[:no_clobber]
|
358
370
|
File.open(filepath,'w') do |file|
|
359
371
|
file.puts lines
|
360
372
|
end
|
361
373
|
end
|
362
|
-
|
363
|
-
puts %Q
|
374
|
+
|
375
|
+
puts %Q("#{filepath}" updated_on: #{update_count})
|
364
376
|
end
|
365
377
|
end
|
366
|
-
|
367
|
-
def log_opts()
|
368
|
-
log.info("Using options#{@options}")
|
369
|
-
end
|
370
|
-
|
371
|
-
def cmd?(cmd)
|
372
|
-
return !@did_cmd && !@cmd.nil?() && @cmd.match?(/\A[[:space:]]*#{Regexp.escape(cmd)}[[:space:]]*\z/i)
|
373
|
-
end
|
374
|
-
|
375
|
-
def do_cmd?()
|
376
|
-
return !@options[:help] && !@options[:version]
|
377
|
-
end
|
378
378
|
end
|
379
379
|
end
|
380
380
|
|
381
|
-
if
|
381
|
+
if IS_SCRIPT
|
382
382
|
main = UncleKryon::Main.new(ARGV)
|
383
|
-
main.run
|
383
|
+
main.run
|
384
384
|
end
|