unclekryon 0.4.9.pre.alpha → 0.4.12.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +10 -26
- data/Gemfile.lock +20 -24
- data/README.md +3 -2
- data/Rakefile +11 -30
- data/bin/unclekryon +4 -15
- data/hax/kryon.yaml +28 -0
- data/hax/kryon_aums_2002-2005.yaml +460 -0
- data/hax/kryon_aums_2006.yaml +601 -0
- data/hax/kryon_aums_2007.yaml +1024 -0
- data/hax/kryon_aums_2008.yaml +950 -0
- data/hax/kryon_aums_2009.yaml +496 -0
- data/hax/kryon_aums_2010.yaml +1443 -0
- data/hax/kryon_aums_2011.yaml +1458 -0
- data/hax/kryon_aums_2012.yaml +2123 -0
- data/hax/kryon_aums_2013.yaml +1647 -0
- data/hax/kryon_aums_2014.yaml +2478 -0
- data/hax/kryon_aums_2015.yaml +3386 -0
- data/hax/kryon_aums_2016.yaml +3476 -0
- data/hax/kryon_aums_2017.yaml +3712 -0
- data/hax/kryon_aums_2018.yaml +3654 -0
- data/lib/unclekryon/data/album_data.rb +74 -82
- data/lib/unclekryon/data/artist_data.rb +24 -36
- data/lib/unclekryon/data/artist_data_data.rb +29 -41
- data/lib/unclekryon/data/aum_data.rb +20 -32
- data/lib/unclekryon/data/base_data.rb +27 -39
- data/lib/unclekryon/data/pic_data.rb +25 -37
- data/lib/unclekryon/data/release_data.rb +14 -26
- data/lib/unclekryon/data/social_data.rb +6 -18
- data/lib/unclekryon/data/timespan_data.rb +16 -28
- data/lib/unclekryon/dev_opts.rb +7 -19
- data/lib/unclekryon/hacker.rb +121 -135
- data/lib/unclekryon/iso/base_iso.rb +69 -81
- data/lib/unclekryon/iso/can_prov_terr.rb +34 -47
- data/lib/unclekryon/iso/country.rb +34 -51
- data/lib/unclekryon/iso/language.rb +84 -98
- data/lib/unclekryon/iso/region.rb +8 -29
- data/lib/unclekryon/iso/subregion.rb +8 -29
- data/lib/unclekryon/iso/usa_state.rb +28 -41
- data/lib/unclekryon/iso.rb +128 -138
- data/lib/unclekryon/jsoner.rb +31 -50
- data/lib/unclekryon/log.rb +34 -46
- data/lib/unclekryon/parsers/kryon_aum_year_album_parser.rb +163 -167
- data/lib/unclekryon/parsers/kryon_aum_year_parser.rb +122 -127
- data/lib/unclekryon/server.rb +8 -17
- data/lib/unclekryon/trainer.rb +68 -85
- data/lib/unclekryon/uploader.rb +8 -17
- data/lib/unclekryon/util.rb +80 -92
- data/lib/unclekryon/version.rb +4 -16
- data/lib/unclekryon.rb +166 -166
- data/train/kryon.yaml +6077 -0
- data/unclekryon.gemspec +49 -49
- metadata +50 -22
data/lib/unclekryon/hacker.rb
CHANGED
@@ -1,28 +1,14 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of UncleKryon-server.
|
7
|
-
# Copyright (c) 2017-
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# UncleKryon-server is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with UncleKryon-server. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2017-2022 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
24
|
-
require 'bundler/setup'
|
25
|
-
|
26
12
|
require 'date'
|
27
13
|
|
28
14
|
require 'unclekryon/log'
|
@@ -36,21 +22,21 @@ require 'unclekryon/parsers/kryon_aum_year_parser'
|
|
36
22
|
module UncleKryon
|
37
23
|
class Hacker
|
38
24
|
include Logging
|
39
|
-
|
25
|
+
|
40
26
|
HAX_DIRNAME = 'hax'
|
41
27
|
HAX_KRYON_FILENAME = 'kryon_<hax>_<release>.yaml'
|
42
|
-
|
28
|
+
|
43
29
|
TRAIN_DIRNAME = 'train'
|
44
30
|
TRAIN_KRYON_FILENAME = 'kryon.yaml'
|
45
|
-
|
31
|
+
|
46
32
|
attr_accessor :hax_dirname
|
47
33
|
attr_accessor :hax_kryon_filename
|
48
34
|
attr_accessor :no_clobber
|
49
35
|
attr_accessor :train_dirname
|
50
36
|
attr_accessor :train_kryon_filename
|
51
|
-
|
37
|
+
|
52
38
|
alias_method :no_clobber?,:no_clobber
|
53
|
-
|
39
|
+
|
54
40
|
def initialize(hax_dirname: HAX_DIRNAME,hax_kryon_filename: HAX_KRYON_FILENAME,no_clobber: false,
|
55
41
|
train_dirname: TRAIN_DIRNAME,train_kryon_filename: TRAIN_KRYON_FILENAME,**options)
|
56
42
|
@hax_dirname = hax_dirname
|
@@ -59,268 +45,268 @@ module UncleKryon
|
|
59
45
|
@train_dirname = train_dirname
|
60
46
|
@train_kryon_filename = train_kryon_filename
|
61
47
|
end
|
62
|
-
|
48
|
+
|
63
49
|
def create_kryon_aum_year_album_parser(date,year=nil,index=nil)
|
64
50
|
pd = parse_date(date,year,index)
|
65
51
|
date = pd[:date]
|
66
52
|
index = pd[:index]
|
67
53
|
year = pd[:year]
|
68
|
-
|
54
|
+
|
69
55
|
# Try the yaml file
|
70
|
-
artist = ArtistDataData.load_file(
|
56
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
71
57
|
release = artist.releases[year]
|
72
|
-
|
73
|
-
if release.nil?
|
58
|
+
|
59
|
+
if release.nil?
|
74
60
|
# Try manually from the site
|
75
61
|
year_parser = create_kryon_aum_year_parser(year)
|
76
62
|
artist = year_parser.artist
|
77
|
-
release = year_parser.parse_site
|
78
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
63
|
+
release = year_parser.parse_site
|
64
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
79
65
|
end
|
80
|
-
|
66
|
+
|
81
67
|
album = find_kryon_aum_year_album(artist,date,year,index)[0]
|
82
68
|
album_parser = KryonAumYearAlbumParser.new(artist,album.url)
|
83
|
-
|
69
|
+
|
84
70
|
album_parser.album = album
|
85
|
-
album_parser.trainers.filepath =
|
86
|
-
|
71
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
72
|
+
|
87
73
|
return album_parser
|
88
74
|
end
|
89
|
-
|
75
|
+
|
90
76
|
def create_kryon_aum_year_parser(year)
|
91
77
|
year_parser = KryonAumYearParser.new(year)
|
92
|
-
|
93
|
-
year_parser.trainers.filepath =
|
94
|
-
|
78
|
+
|
79
|
+
year_parser.trainers.filepath = build_train_kryon_filepath
|
80
|
+
|
95
81
|
return year_parser
|
96
82
|
end
|
97
|
-
|
83
|
+
|
98
84
|
def find_kryon_aum_year_album(artist,date,year=nil,index=nil)
|
99
85
|
album = nil
|
100
86
|
albums = []
|
101
|
-
|
87
|
+
|
102
88
|
artist.albums.values.each_with_index do |a,i|
|
103
89
|
date_begin = Util.parse_date_s(a.date_begin)
|
104
90
|
date_end = Util.parse_date_s(a.date_end)
|
105
|
-
|
91
|
+
|
106
92
|
if (date_begin && ((date_end && date >= date_begin && date <= date_end) ||
|
107
93
|
(!date_end && date == date_begin)))
|
108
94
|
albums.push([a,i])
|
109
95
|
end
|
110
96
|
end
|
111
|
-
|
112
|
-
if !albums.empty?
|
97
|
+
|
98
|
+
if !albums.empty?
|
113
99
|
if index >= 0 && index < albums.length
|
114
100
|
album = albums[index]
|
115
101
|
else
|
116
102
|
raise "Invalid album ordinal number[#{index + 1}]"
|
117
103
|
end
|
118
104
|
end
|
119
|
-
|
120
|
-
raise "Invalid album[#{date}]" if album.nil?
|
121
|
-
|
105
|
+
|
106
|
+
raise "Invalid album[#{date}]" if album.nil?
|
107
|
+
|
122
108
|
return album
|
123
109
|
end
|
124
|
-
|
110
|
+
|
125
111
|
def parse_date(date,year=nil,index=nil)
|
126
112
|
if !date.is_a?(Date)
|
127
113
|
ds = date.split(':')
|
128
114
|
date = ds[0]
|
129
|
-
|
130
|
-
if index.nil?
|
115
|
+
|
116
|
+
if index.nil?
|
131
117
|
index = ds
|
132
|
-
index = (index.length <= 1) ? 0 : (index[1].to_i
|
118
|
+
index = (index.length <= 1) ? 0 : (index[1].to_i - 1)
|
133
119
|
end
|
134
|
-
|
120
|
+
|
135
121
|
begin
|
136
122
|
new_date = Date.strptime(date,'%Y.%m.%d')
|
137
123
|
rescue ArgumentError
|
138
124
|
new_date = Date.strptime(date,'%m.%d')
|
139
|
-
|
140
|
-
if !year.nil?
|
141
|
-
new_date = Date.new(year.to_i
|
125
|
+
|
126
|
+
if !year.nil?
|
127
|
+
new_date = Date.new(year.to_i,new_date.month,new_date.day)
|
142
128
|
end
|
143
129
|
end
|
144
|
-
|
130
|
+
|
145
131
|
date = new_date
|
146
|
-
elsif index.nil?
|
132
|
+
elsif index.nil?
|
147
133
|
index = 0
|
148
134
|
end
|
149
|
-
|
150
|
-
if year.nil?
|
135
|
+
|
136
|
+
if year.nil?
|
151
137
|
# year is actually the release's title, so only override it if have to
|
152
|
-
year = date.year.to_s
|
138
|
+
year = date.year.to_s
|
153
139
|
end
|
154
|
-
|
155
|
-
return {:date
|
140
|
+
|
141
|
+
return {date: date,index: index,year: year}
|
156
142
|
end
|
157
|
-
|
143
|
+
|
158
144
|
def parse_kryon_aum_year(year)
|
159
145
|
year_parser = create_kryon_aum_year_parser(year)
|
160
|
-
release = year_parser.parse_site
|
161
|
-
|
146
|
+
release = year_parser.parse_site
|
147
|
+
|
162
148
|
if @no_clobber
|
163
|
-
puts release.to_s
|
149
|
+
puts release.to_s
|
164
150
|
else
|
165
|
-
year_parser.artist.save_to_file(
|
151
|
+
year_parser.artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
166
152
|
end
|
167
153
|
end
|
168
|
-
|
154
|
+
|
169
155
|
def parse_kryon_aum_year_album(date,year=nil,index=nil)
|
170
156
|
pd = parse_date(date,year,index)
|
171
157
|
date = pd[:date]
|
172
158
|
index = pd[:index]
|
173
159
|
year = pd[:year]
|
174
|
-
|
160
|
+
|
175
161
|
album_parser = create_kryon_aum_year_album_parser(date,year,index)
|
176
|
-
album_parser.parse_site
|
177
|
-
|
162
|
+
album_parser.parse_site
|
163
|
+
|
178
164
|
if @no_clobber
|
179
|
-
puts album_parser.album.to_s
|
165
|
+
puts album_parser.album.to_s
|
180
166
|
else
|
181
|
-
album_parser.artist.save_to_file(
|
167
|
+
album_parser.artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
182
168
|
end
|
183
169
|
end
|
184
|
-
|
170
|
+
|
185
171
|
def parse_kryon_aum_year_albums(year,begin_album=nil)
|
186
|
-
if !begin_album.nil?
|
172
|
+
if !begin_album.nil?
|
187
173
|
pd = parse_date(begin_album,year)
|
188
174
|
begin_album = pd[:date]
|
189
175
|
index = pd[:index]
|
190
176
|
year = pd[:year]
|
191
177
|
end
|
192
|
-
|
178
|
+
|
193
179
|
# Try the yaml file
|
194
|
-
artist = ArtistDataData.load_file(
|
180
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
195
181
|
release = artist.releases[year]
|
196
182
|
updated_on = nil
|
197
|
-
|
198
|
-
if release.nil?
|
183
|
+
|
184
|
+
if release.nil?
|
199
185
|
# Try manually from the site
|
200
186
|
year_parser = create_kryon_aum_year_parser(year)
|
201
187
|
artist = year_parser.artist
|
202
|
-
release = year_parser.parse_site
|
203
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
188
|
+
release = year_parser.parse_site
|
189
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
204
190
|
updated_on = release.updated_on
|
205
191
|
end
|
206
|
-
|
192
|
+
|
207
193
|
album_parser = KryonAumYearAlbumParser.new
|
208
|
-
album_parser.trainers.filepath =
|
209
|
-
album_parser.updated_on = updated_on unless updated_on.nil?
|
210
|
-
|
194
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
195
|
+
album_parser.updated_on = updated_on unless updated_on.nil?
|
196
|
+
|
211
197
|
albums = release.albums
|
212
|
-
|
213
|
-
if !begin_album.nil?
|
198
|
+
|
199
|
+
if !begin_album.nil?
|
214
200
|
album_index = find_kryon_aum_year_album(artist,begin_album,year,index)[1]
|
215
|
-
albums = albums[album_index
|
201
|
+
albums = albums[album_index..]
|
216
202
|
end
|
217
|
-
|
203
|
+
|
218
204
|
albums.each do |album_id|
|
219
205
|
album = artist.albums[album_id]
|
220
206
|
log.info("Hacking album[#{album.date_begin},#{album.date_end},#{album.title}]")
|
221
|
-
album = album_parser.parse_site(artist,album.url)
|
207
|
+
#album = album_parser.parse_site(artist,album.url)
|
222
208
|
end
|
223
|
-
|
209
|
+
|
224
210
|
if @no_clobber
|
225
|
-
puts release.to_s
|
211
|
+
puts release.to_s
|
226
212
|
else
|
227
|
-
artist.save_to_file(
|
213
|
+
artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
228
214
|
end
|
229
215
|
end
|
230
|
-
|
216
|
+
|
231
217
|
def train_kryon_aum_year(year)
|
232
218
|
year_parser = create_kryon_aum_year_parser(year)
|
233
219
|
year_parser.training = true
|
234
|
-
year_parser.parse_site
|
235
|
-
|
220
|
+
year_parser.parse_site
|
221
|
+
|
236
222
|
if @no_clobber
|
237
|
-
puts year_parser.trainers.to_s
|
223
|
+
puts year_parser.trainers.to_s
|
238
224
|
else
|
239
|
-
year_parser.trainers.save_to_file
|
225
|
+
year_parser.trainers.save_to_file
|
240
226
|
end
|
241
227
|
end
|
242
|
-
|
228
|
+
|
243
229
|
def train_kryon_aum_year_album(date,year=nil,index=nil)
|
244
230
|
album_parser = create_kryon_aum_year_album_parser(date,year,index)
|
245
231
|
album_parser.training = true
|
246
|
-
album_parser.parse_site
|
247
|
-
|
232
|
+
album_parser.parse_site
|
233
|
+
|
248
234
|
if @no_clobber
|
249
|
-
puts album_parser.trainers.to_s
|
235
|
+
puts album_parser.trainers.to_s
|
250
236
|
else
|
251
|
-
album_parser.trainers.save_to_file
|
237
|
+
album_parser.trainers.save_to_file
|
252
238
|
end
|
253
239
|
end
|
254
|
-
|
240
|
+
|
255
241
|
def train_kryon_aum_year_albums(year,begin_album=nil)
|
256
|
-
if !begin_album.nil?
|
242
|
+
if !begin_album.nil?
|
257
243
|
pd = parse_date(begin_album,year)
|
258
244
|
begin_album = pd[:date]
|
259
245
|
index = pd[:index]
|
260
246
|
year = pd[:year]
|
261
247
|
end
|
262
|
-
|
248
|
+
|
263
249
|
# Try the yaml file
|
264
|
-
artist = ArtistDataData.load_file(
|
250
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
265
251
|
release = artist.releases[year]
|
266
|
-
|
267
|
-
if release.nil?
|
252
|
+
|
253
|
+
if release.nil?
|
268
254
|
# Try manually from the site
|
269
255
|
year_parser = create_kryon_aum_year_parser(year)
|
270
256
|
artist = year_parser.artist
|
271
|
-
release = year_parser.parse_site
|
272
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
257
|
+
release = year_parser.parse_site
|
258
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
273
259
|
end
|
274
|
-
|
260
|
+
|
275
261
|
albums = release.albums
|
276
|
-
|
277
|
-
if !begin_album.nil?
|
262
|
+
|
263
|
+
if !begin_album.nil?
|
278
264
|
album_index = find_kryon_aum_year_album(artist,begin_album,year,index)[1]
|
279
|
-
albums = albums[album_index
|
265
|
+
albums = albums[album_index..]
|
280
266
|
end
|
281
|
-
|
267
|
+
|
282
268
|
albums.each do |album_id|
|
283
269
|
album = artist.albums[album_id]
|
284
|
-
|
270
|
+
|
285
271
|
album_parser = KryonAumYearAlbumParser.new
|
286
272
|
album_parser.album = album
|
287
|
-
album_parser.trainers.filepath =
|
273
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
288
274
|
album_parser.training = true
|
289
|
-
|
275
|
+
|
290
276
|
log.info("Training album[#{album.date_begin},#{album.date_end},#{album.title}]")
|
291
|
-
album = album_parser.parse_site(artist,album.url)
|
292
|
-
|
277
|
+
#album = album_parser.parse_site(artist,album.url)
|
278
|
+
|
293
279
|
if @no_clobber
|
294
|
-
puts album_parser.trainers.to_s
|
280
|
+
puts album_parser.trainers.to_s
|
295
281
|
else
|
296
|
-
album_parser.trainers.save_to_file
|
282
|
+
album_parser.trainers.save_to_file
|
297
283
|
end
|
298
284
|
end
|
299
285
|
end
|
300
|
-
|
301
|
-
def
|
302
|
-
return
|
286
|
+
|
287
|
+
def build_hax_kryon_aums_filepath(release)
|
288
|
+
return build_hax_kryon_filepath('aums',release)
|
303
289
|
end
|
304
|
-
|
305
|
-
def
|
306
|
-
raise
|
307
|
-
|
308
|
-
fn = @hax_kryon_filename.clone
|
290
|
+
|
291
|
+
def build_hax_kryon_filepath(hax,release)
|
292
|
+
raise 'Release (year) arg is nil' if release.nil?
|
293
|
+
|
294
|
+
fn = @hax_kryon_filename.clone
|
309
295
|
fn = fn.gsub('<hax>',hax)
|
310
|
-
fn = fn.gsub('<release>',release.to_s
|
311
|
-
|
296
|
+
fn = fn.gsub('<release>',release.to_s)
|
297
|
+
|
312
298
|
return File.join(@hax_dirname,fn)
|
313
299
|
end
|
314
|
-
|
315
|
-
def
|
300
|
+
|
301
|
+
def build_train_kryon_filepath
|
316
302
|
return File.join(@train_dirname,@train_kryon_filename)
|
317
303
|
end
|
318
304
|
end
|
319
305
|
end
|
320
306
|
|
321
|
-
if $
|
307
|
+
if $PROGRAM_NAME == __FILE__
|
322
308
|
hacker = UncleKryon::Hacker.new(no_clobber: true)
|
323
|
-
|
309
|
+
|
324
310
|
#hacker.parse_kryon_aum_year('2017')
|
325
311
|
#hacker.parse_kryon_aum_year_albums('2017')
|
326
312
|
hacker.train_kryon_aum_year_album('2.2','2017')
|