unclekryon 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +6 -19
- data/Gemfile.lock +19 -23
- data/README.md +3 -2
- data/Rakefile +11 -30
- data/bin/unclekryon +4 -15
- data/hax/kryon.yaml +28 -0
- data/hax/kryon_aums_2002-2005.yaml +460 -0
- data/hax/kryon_aums_2006.yaml +601 -0
- data/hax/kryon_aums_2007.yaml +1024 -0
- data/hax/kryon_aums_2008.yaml +950 -0
- data/hax/kryon_aums_2009.yaml +496 -0
- data/hax/kryon_aums_2010.yaml +1443 -0
- data/hax/kryon_aums_2011.yaml +1458 -0
- data/hax/kryon_aums_2012.yaml +2123 -0
- data/hax/kryon_aums_2013.yaml +1647 -0
- data/hax/kryon_aums_2014.yaml +2478 -0
- data/hax/kryon_aums_2015.yaml +3386 -0
- data/hax/kryon_aums_2016.yaml +3476 -0
- data/hax/kryon_aums_2017.yaml +3712 -0
- data/hax/kryon_aums_2018.yaml +3654 -0
- data/lib/unclekryon.rb +165 -165
- data/lib/unclekryon/data/album_data.rb +74 -82
- data/lib/unclekryon/data/artist_data.rb +24 -36
- data/lib/unclekryon/data/artist_data_data.rb +29 -41
- data/lib/unclekryon/data/aum_data.rb +20 -32
- data/lib/unclekryon/data/base_data.rb +27 -39
- data/lib/unclekryon/data/pic_data.rb +25 -37
- data/lib/unclekryon/data/release_data.rb +14 -26
- data/lib/unclekryon/data/social_data.rb +6 -18
- data/lib/unclekryon/data/timespan_data.rb +16 -28
- data/lib/unclekryon/dev_opts.rb +7 -19
- data/lib/unclekryon/hacker.rb +119 -133
- data/lib/unclekryon/iso.rb +128 -138
- data/lib/unclekryon/iso/base_iso.rb +69 -81
- data/lib/unclekryon/iso/can_prov_terr.rb +34 -47
- data/lib/unclekryon/iso/country.rb +36 -49
- data/lib/unclekryon/iso/language.rb +86 -96
- data/lib/unclekryon/iso/region.rb +11 -25
- data/lib/unclekryon/iso/subregion.rb +11 -25
- data/lib/unclekryon/iso/usa_state.rb +28 -41
- data/lib/unclekryon/jsoner.rb +31 -50
- data/lib/unclekryon/log.rb +34 -46
- data/lib/unclekryon/parsers/kryon_aum_year_album_parser.rb +163 -167
- data/lib/unclekryon/parsers/kryon_aum_year_parser.rb +122 -127
- data/lib/unclekryon/server.rb +8 -17
- data/lib/unclekryon/trainer.rb +69 -83
- data/lib/unclekryon/uploader.rb +8 -17
- data/lib/unclekryon/util.rb +80 -92
- data/lib/unclekryon/version.rb +4 -16
- data/train/kryon.yaml +6077 -0
- data/unclekryon.gemspec +44 -42
- metadata +59 -16
data/lib/unclekryon/dev_opts.rb
CHANGED
@@ -1,23 +1,11 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of UncleKryon-server.
|
7
|
-
# Copyright (c) 2018-
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# UncleKryon-server is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with UncleKryon-server. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2018-2021 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
@@ -26,14 +14,14 @@ require 'singleton'
|
|
26
14
|
module UncleKryon
|
27
15
|
class DevOpts
|
28
16
|
include Singleton
|
29
|
-
|
17
|
+
|
30
18
|
attr_accessor :dev
|
31
19
|
attr_accessor :test
|
32
|
-
|
20
|
+
|
33
21
|
alias_method :dev?,:dev
|
34
22
|
alias_method :test?,:test
|
35
|
-
|
36
|
-
def initialize
|
23
|
+
|
24
|
+
def initialize
|
37
25
|
@dev = false
|
38
26
|
@test = false
|
39
27
|
end
|
data/lib/unclekryon/hacker.rb
CHANGED
@@ -1,28 +1,14 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
1
|
# encoding: UTF-8
|
3
2
|
# frozen_string_literal: true
|
4
3
|
|
5
4
|
#--
|
6
5
|
# This file is part of UncleKryon-server.
|
7
|
-
# Copyright (c) 2017-
|
8
|
-
#
|
9
|
-
#
|
10
|
-
# it under the terms of the GNU General Public License as published by
|
11
|
-
# the Free Software Foundation, either version 3 of the License, or
|
12
|
-
# (at your option) any later version.
|
13
|
-
#
|
14
|
-
# UncleKryon-server is distributed in the hope that it will be useful,
|
15
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
16
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
17
|
-
# GNU General Public License for more details.
|
18
|
-
#
|
19
|
-
# You should have received a copy of the GNU General Public License
|
20
|
-
# along with UncleKryon-server. If not, see <https://www.gnu.org/licenses/>.
|
6
|
+
# Copyright (c) 2017-2021 Jonathan Bradley Whited
|
7
|
+
#
|
8
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
21
9
|
#++
|
22
10
|
|
23
11
|
|
24
|
-
require 'bundler/setup'
|
25
|
-
|
26
12
|
require 'date'
|
27
13
|
|
28
14
|
require 'unclekryon/log'
|
@@ -36,21 +22,21 @@ require 'unclekryon/parsers/kryon_aum_year_parser'
|
|
36
22
|
module UncleKryon
|
37
23
|
class Hacker
|
38
24
|
include Logging
|
39
|
-
|
25
|
+
|
40
26
|
HAX_DIRNAME = 'hax'
|
41
27
|
HAX_KRYON_FILENAME = 'kryon_<hax>_<release>.yaml'
|
42
|
-
|
28
|
+
|
43
29
|
TRAIN_DIRNAME = 'train'
|
44
30
|
TRAIN_KRYON_FILENAME = 'kryon.yaml'
|
45
|
-
|
31
|
+
|
46
32
|
attr_accessor :hax_dirname
|
47
33
|
attr_accessor :hax_kryon_filename
|
48
34
|
attr_accessor :no_clobber
|
49
35
|
attr_accessor :train_dirname
|
50
36
|
attr_accessor :train_kryon_filename
|
51
|
-
|
37
|
+
|
52
38
|
alias_method :no_clobber?,:no_clobber
|
53
|
-
|
39
|
+
|
54
40
|
def initialize(hax_dirname: HAX_DIRNAME,hax_kryon_filename: HAX_KRYON_FILENAME,no_clobber: false,
|
55
41
|
train_dirname: TRAIN_DIRNAME,train_kryon_filename: TRAIN_KRYON_FILENAME,**options)
|
56
42
|
@hax_dirname = hax_dirname
|
@@ -59,268 +45,268 @@ module UncleKryon
|
|
59
45
|
@train_dirname = train_dirname
|
60
46
|
@train_kryon_filename = train_kryon_filename
|
61
47
|
end
|
62
|
-
|
48
|
+
|
63
49
|
def create_kryon_aum_year_album_parser(date,year=nil,index=nil)
|
64
50
|
pd = parse_date(date,year,index)
|
65
51
|
date = pd[:date]
|
66
52
|
index = pd[:index]
|
67
53
|
year = pd[:year]
|
68
|
-
|
54
|
+
|
69
55
|
# Try the yaml file
|
70
|
-
artist = ArtistDataData.load_file(
|
56
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
71
57
|
release = artist.releases[year]
|
72
|
-
|
73
|
-
if release.nil?
|
58
|
+
|
59
|
+
if release.nil?
|
74
60
|
# Try manually from the site
|
75
61
|
year_parser = create_kryon_aum_year_parser(year)
|
76
62
|
artist = year_parser.artist
|
77
|
-
release = year_parser.parse_site
|
78
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
63
|
+
release = year_parser.parse_site
|
64
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
79
65
|
end
|
80
|
-
|
66
|
+
|
81
67
|
album = find_kryon_aum_year_album(artist,date,year,index)[0]
|
82
68
|
album_parser = KryonAumYearAlbumParser.new(artist,album.url)
|
83
|
-
|
69
|
+
|
84
70
|
album_parser.album = album
|
85
|
-
album_parser.trainers.filepath =
|
86
|
-
|
71
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
72
|
+
|
87
73
|
return album_parser
|
88
74
|
end
|
89
|
-
|
75
|
+
|
90
76
|
def create_kryon_aum_year_parser(year)
|
91
77
|
year_parser = KryonAumYearParser.new(year)
|
92
|
-
|
93
|
-
year_parser.trainers.filepath =
|
94
|
-
|
78
|
+
|
79
|
+
year_parser.trainers.filepath = build_train_kryon_filepath
|
80
|
+
|
95
81
|
return year_parser
|
96
82
|
end
|
97
|
-
|
83
|
+
|
98
84
|
def find_kryon_aum_year_album(artist,date,year=nil,index=nil)
|
99
85
|
album = nil
|
100
86
|
albums = []
|
101
|
-
|
87
|
+
|
102
88
|
artist.albums.values.each_with_index do |a,i|
|
103
89
|
date_begin = Util.parse_date_s(a.date_begin)
|
104
90
|
date_end = Util.parse_date_s(a.date_end)
|
105
|
-
|
91
|
+
|
106
92
|
if (date_begin && ((date_end && date >= date_begin && date <= date_end) ||
|
107
93
|
(!date_end && date == date_begin)))
|
108
94
|
albums.push([a,i])
|
109
95
|
end
|
110
96
|
end
|
111
|
-
|
112
|
-
if !albums.empty?
|
97
|
+
|
98
|
+
if !albums.empty?
|
113
99
|
if index >= 0 && index < albums.length
|
114
100
|
album = albums[index]
|
115
101
|
else
|
116
102
|
raise "Invalid album ordinal number[#{index + 1}]"
|
117
103
|
end
|
118
104
|
end
|
119
|
-
|
120
|
-
raise "Invalid album[#{date}]" if album.nil?
|
121
|
-
|
105
|
+
|
106
|
+
raise "Invalid album[#{date}]" if album.nil?
|
107
|
+
|
122
108
|
return album
|
123
109
|
end
|
124
|
-
|
110
|
+
|
125
111
|
def parse_date(date,year=nil,index=nil)
|
126
112
|
if !date.is_a?(Date)
|
127
113
|
ds = date.split(':')
|
128
114
|
date = ds[0]
|
129
|
-
|
130
|
-
if index.nil?
|
115
|
+
|
116
|
+
if index.nil?
|
131
117
|
index = ds
|
132
|
-
index = (index.length <= 1) ? 0 : (index[1].to_i
|
118
|
+
index = (index.length <= 1) ? 0 : (index[1].to_i - 1)
|
133
119
|
end
|
134
|
-
|
120
|
+
|
135
121
|
begin
|
136
122
|
new_date = Date.strptime(date,'%Y.%m.%d')
|
137
123
|
rescue ArgumentError
|
138
124
|
new_date = Date.strptime(date,'%m.%d')
|
139
|
-
|
140
|
-
if !year.nil?
|
141
|
-
new_date = Date.new(year.to_i
|
125
|
+
|
126
|
+
if !year.nil?
|
127
|
+
new_date = Date.new(year.to_i,new_date.month,new_date.day)
|
142
128
|
end
|
143
129
|
end
|
144
|
-
|
130
|
+
|
145
131
|
date = new_date
|
146
|
-
elsif index.nil?
|
132
|
+
elsif index.nil?
|
147
133
|
index = 0
|
148
134
|
end
|
149
|
-
|
150
|
-
if year.nil?
|
135
|
+
|
136
|
+
if year.nil?
|
151
137
|
# year is actually the release's title, so only override it if have to
|
152
|
-
year = date.year.to_s
|
138
|
+
year = date.year.to_s
|
153
139
|
end
|
154
|
-
|
155
|
-
return {:date
|
140
|
+
|
141
|
+
return {date: date,index: index,year: year}
|
156
142
|
end
|
157
|
-
|
143
|
+
|
158
144
|
def parse_kryon_aum_year(year)
|
159
145
|
year_parser = create_kryon_aum_year_parser(year)
|
160
|
-
release = year_parser.parse_site
|
161
|
-
|
146
|
+
release = year_parser.parse_site
|
147
|
+
|
162
148
|
if @no_clobber
|
163
|
-
puts release.to_s
|
149
|
+
puts release.to_s
|
164
150
|
else
|
165
|
-
year_parser.artist.save_to_file(
|
151
|
+
year_parser.artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
166
152
|
end
|
167
153
|
end
|
168
|
-
|
154
|
+
|
169
155
|
def parse_kryon_aum_year_album(date,year=nil,index=nil)
|
170
156
|
pd = parse_date(date,year,index)
|
171
157
|
date = pd[:date]
|
172
158
|
index = pd[:index]
|
173
159
|
year = pd[:year]
|
174
|
-
|
160
|
+
|
175
161
|
album_parser = create_kryon_aum_year_album_parser(date,year,index)
|
176
|
-
album_parser.parse_site
|
177
|
-
|
162
|
+
album_parser.parse_site
|
163
|
+
|
178
164
|
if @no_clobber
|
179
|
-
puts album_parser.album.to_s
|
165
|
+
puts album_parser.album.to_s
|
180
166
|
else
|
181
|
-
album_parser.artist.save_to_file(
|
167
|
+
album_parser.artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
182
168
|
end
|
183
169
|
end
|
184
|
-
|
170
|
+
|
185
171
|
def parse_kryon_aum_year_albums(year,begin_album=nil)
|
186
|
-
if !begin_album.nil?
|
172
|
+
if !begin_album.nil?
|
187
173
|
pd = parse_date(begin_album,year)
|
188
174
|
begin_album = pd[:date]
|
189
175
|
index = pd[:index]
|
190
176
|
year = pd[:year]
|
191
177
|
end
|
192
|
-
|
178
|
+
|
193
179
|
# Try the yaml file
|
194
|
-
artist = ArtistDataData.load_file(
|
180
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
195
181
|
release = artist.releases[year]
|
196
182
|
updated_on = nil
|
197
|
-
|
198
|
-
if release.nil?
|
183
|
+
|
184
|
+
if release.nil?
|
199
185
|
# Try manually from the site
|
200
186
|
year_parser = create_kryon_aum_year_parser(year)
|
201
187
|
artist = year_parser.artist
|
202
|
-
release = year_parser.parse_site
|
203
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
188
|
+
release = year_parser.parse_site
|
189
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
204
190
|
updated_on = release.updated_on
|
205
191
|
end
|
206
|
-
|
192
|
+
|
207
193
|
album_parser = KryonAumYearAlbumParser.new
|
208
|
-
album_parser.trainers.filepath =
|
209
|
-
album_parser.updated_on = updated_on unless updated_on.nil?
|
210
|
-
|
194
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
195
|
+
album_parser.updated_on = updated_on unless updated_on.nil?
|
196
|
+
|
211
197
|
albums = release.albums
|
212
|
-
|
213
|
-
if !begin_album.nil?
|
198
|
+
|
199
|
+
if !begin_album.nil?
|
214
200
|
album_index = find_kryon_aum_year_album(artist,begin_album,year,index)[1]
|
215
201
|
albums = albums[album_index..-1]
|
216
202
|
end
|
217
|
-
|
203
|
+
|
218
204
|
albums.each do |album_id|
|
219
205
|
album = artist.albums[album_id]
|
220
206
|
log.info("Hacking album[#{album.date_begin},#{album.date_end},#{album.title}]")
|
221
|
-
album = album_parser.parse_site(artist,album.url)
|
207
|
+
#album = album_parser.parse_site(artist,album.url)
|
222
208
|
end
|
223
|
-
|
209
|
+
|
224
210
|
if @no_clobber
|
225
|
-
puts release.to_s
|
211
|
+
puts release.to_s
|
226
212
|
else
|
227
|
-
artist.save_to_file(
|
213
|
+
artist.save_to_file(build_hax_kryon_aums_filepath(year))
|
228
214
|
end
|
229
215
|
end
|
230
|
-
|
216
|
+
|
231
217
|
def train_kryon_aum_year(year)
|
232
218
|
year_parser = create_kryon_aum_year_parser(year)
|
233
219
|
year_parser.training = true
|
234
|
-
year_parser.parse_site
|
235
|
-
|
220
|
+
year_parser.parse_site
|
221
|
+
|
236
222
|
if @no_clobber
|
237
|
-
puts year_parser.trainers.to_s
|
223
|
+
puts year_parser.trainers.to_s
|
238
224
|
else
|
239
|
-
year_parser.trainers.save_to_file
|
225
|
+
year_parser.trainers.save_to_file
|
240
226
|
end
|
241
227
|
end
|
242
|
-
|
228
|
+
|
243
229
|
def train_kryon_aum_year_album(date,year=nil,index=nil)
|
244
230
|
album_parser = create_kryon_aum_year_album_parser(date,year,index)
|
245
231
|
album_parser.training = true
|
246
|
-
album_parser.parse_site
|
247
|
-
|
232
|
+
album_parser.parse_site
|
233
|
+
|
248
234
|
if @no_clobber
|
249
|
-
puts album_parser.trainers.to_s
|
235
|
+
puts album_parser.trainers.to_s
|
250
236
|
else
|
251
|
-
album_parser.trainers.save_to_file
|
237
|
+
album_parser.trainers.save_to_file
|
252
238
|
end
|
253
239
|
end
|
254
|
-
|
240
|
+
|
255
241
|
def train_kryon_aum_year_albums(year,begin_album=nil)
|
256
|
-
if !begin_album.nil?
|
242
|
+
if !begin_album.nil?
|
257
243
|
pd = parse_date(begin_album,year)
|
258
244
|
begin_album = pd[:date]
|
259
245
|
index = pd[:index]
|
260
246
|
year = pd[:year]
|
261
247
|
end
|
262
|
-
|
248
|
+
|
263
249
|
# Try the yaml file
|
264
|
-
artist = ArtistDataData.load_file(
|
250
|
+
artist = ArtistDataData.load_file(build_hax_kryon_aums_filepath(year))
|
265
251
|
release = artist.releases[year]
|
266
|
-
|
267
|
-
if release.nil?
|
252
|
+
|
253
|
+
if release.nil?
|
268
254
|
# Try manually from the site
|
269
255
|
year_parser = create_kryon_aum_year_parser(year)
|
270
256
|
artist = year_parser.artist
|
271
|
-
release = year_parser.parse_site
|
272
|
-
raise "Release[#{year}] does not exist" if release.nil?
|
257
|
+
release = year_parser.parse_site
|
258
|
+
raise "Release[#{year}] does not exist" if release.nil?
|
273
259
|
end
|
274
|
-
|
260
|
+
|
275
261
|
albums = release.albums
|
276
|
-
|
277
|
-
if !begin_album.nil?
|
262
|
+
|
263
|
+
if !begin_album.nil?
|
278
264
|
album_index = find_kryon_aum_year_album(artist,begin_album,year,index)[1]
|
279
265
|
albums = albums[album_index..-1]
|
280
266
|
end
|
281
|
-
|
267
|
+
|
282
268
|
albums.each do |album_id|
|
283
269
|
album = artist.albums[album_id]
|
284
|
-
|
270
|
+
|
285
271
|
album_parser = KryonAumYearAlbumParser.new
|
286
272
|
album_parser.album = album
|
287
|
-
album_parser.trainers.filepath =
|
273
|
+
album_parser.trainers.filepath = build_train_kryon_filepath
|
288
274
|
album_parser.training = true
|
289
|
-
|
275
|
+
|
290
276
|
log.info("Training album[#{album.date_begin},#{album.date_end},#{album.title}]")
|
291
|
-
album = album_parser.parse_site(artist,album.url)
|
292
|
-
|
277
|
+
#album = album_parser.parse_site(artist,album.url)
|
278
|
+
|
293
279
|
if @no_clobber
|
294
|
-
puts album_parser.trainers.to_s
|
280
|
+
puts album_parser.trainers.to_s
|
295
281
|
else
|
296
|
-
album_parser.trainers.save_to_file
|
282
|
+
album_parser.trainers.save_to_file
|
297
283
|
end
|
298
284
|
end
|
299
285
|
end
|
300
|
-
|
301
|
-
def
|
302
|
-
return
|
286
|
+
|
287
|
+
def build_hax_kryon_aums_filepath(release)
|
288
|
+
return build_hax_kryon_filepath('aums',release)
|
303
289
|
end
|
304
|
-
|
305
|
-
def
|
306
|
-
raise
|
307
|
-
|
308
|
-
fn = @hax_kryon_filename.clone
|
290
|
+
|
291
|
+
def build_hax_kryon_filepath(hax,release)
|
292
|
+
raise 'Release (year) arg is nil' if release.nil?
|
293
|
+
|
294
|
+
fn = @hax_kryon_filename.clone
|
309
295
|
fn = fn.gsub('<hax>',hax)
|
310
|
-
fn = fn.gsub('<release>',release.to_s
|
311
|
-
|
296
|
+
fn = fn.gsub('<release>',release.to_s)
|
297
|
+
|
312
298
|
return File.join(@hax_dirname,fn)
|
313
299
|
end
|
314
|
-
|
315
|
-
def
|
300
|
+
|
301
|
+
def build_train_kryon_filepath
|
316
302
|
return File.join(@train_dirname,@train_kryon_filename)
|
317
303
|
end
|
318
304
|
end
|
319
305
|
end
|
320
306
|
|
321
|
-
if $
|
307
|
+
if $PROGRAM_NAME == __FILE__
|
322
308
|
hacker = UncleKryon::Hacker.new(no_clobber: true)
|
323
|
-
|
309
|
+
|
324
310
|
#hacker.parse_kryon_aum_year('2017')
|
325
311
|
#hacker.parse_kryon_aum_year_albums('2017')
|
326
312
|
hacker.train_kryon_aum_year_album('2.2','2017')
|