metal_archives 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +59 -12
- data/.rspec +1 -0
- data/.rubocop.yml +34 -20
- data/CHANGELOG.md +4 -0
- data/LICENSE.md +17 -4
- data/README.md +29 -14
- data/bin/console +8 -11
- data/config/inflections.rb +7 -0
- data/config/initializers/.keep +0 -0
- data/docker-compose.yml +10 -1
- data/lib/metal_archives.rb +56 -22
- data/lib/metal_archives/cache/base.rb +40 -0
- data/lib/metal_archives/cache/memory.rb +68 -0
- data/lib/metal_archives/cache/null.rb +22 -0
- data/lib/metal_archives/cache/redis.rb +49 -0
- data/lib/metal_archives/collection.rb +3 -5
- data/lib/metal_archives/configuration.rb +28 -21
- data/lib/metal_archives/errors.rb +9 -1
- data/lib/metal_archives/http_client.rb +42 -46
- data/lib/metal_archives/models/artist.rb +55 -26
- data/lib/metal_archives/models/band.rb +43 -36
- data/lib/metal_archives/models/{base_model.rb → base.rb} +53 -53
- data/lib/metal_archives/models/label.rb +7 -8
- data/lib/metal_archives/models/release.rb +11 -17
- data/lib/metal_archives/parsers/artist.rb +40 -35
- data/lib/metal_archives/parsers/band.rb +73 -29
- data/lib/metal_archives/parsers/base.rb +14 -0
- data/lib/metal_archives/parsers/country.rb +21 -0
- data/lib/metal_archives/parsers/date.rb +31 -0
- data/lib/metal_archives/parsers/genre.rb +67 -0
- data/lib/metal_archives/parsers/label.rb +21 -13
- data/lib/metal_archives/parsers/parser.rb +15 -77
- data/lib/metal_archives/parsers/release.rb +22 -18
- data/lib/metal_archives/parsers/year.rb +29 -0
- data/lib/metal_archives/version.rb +3 -3
- data/metal_archives.env.example +7 -4
- data/metal_archives.gemspec +7 -4
- data/nginx/default.conf +2 -2
- metadata +76 -32
- data/.github/workflows/release.yml +0 -69
- data/.rubocop_todo.yml +0 -92
- data/lib/metal_archives/lru_cache.rb +0 -61
- data/lib/metal_archives/middleware/cache_check.rb +0 -18
- data/lib/metal_archives/middleware/encoding.rb +0 -16
- data/lib/metal_archives/middleware/headers.rb +0 -38
- data/lib/metal_archives/middleware/rewrite_endpoint.rb +0 -38
- data/lib/metal_archives/nil_date.rb +0 -91
- data/lib/metal_archives/range.rb +0 -69
@@ -7,7 +7,7 @@ module MetalArchives
|
|
7
7
|
##
|
8
8
|
# Represents a release
|
9
9
|
#
|
10
|
-
class Release <
|
10
|
+
class Release < Base
|
11
11
|
##
|
12
12
|
# :attr_reader: id
|
13
13
|
#
|
@@ -37,18 +37,18 @@ module MetalArchives
|
|
37
37
|
# - rdoc-ref:MetalArchives::Errors::InvalidIDError when no or invalid id
|
38
38
|
# - rdoc-ref:MetalArchives::Errors::APIError when receiving a status code >= 400 (except 404)
|
39
39
|
#
|
40
|
-
enum :type, values:
|
40
|
+
enum :type, values: [:full_length, :live, :demo, :single, :ep, :video, :boxed_set, :split, :compilation, :split_video, :collaboration]
|
41
41
|
|
42
42
|
##
|
43
43
|
# :attr_reader: date_released
|
44
44
|
#
|
45
|
-
# Returns rdoc-ref:
|
45
|
+
# Returns rdoc-ref:Date
|
46
46
|
#
|
47
47
|
# [Raises]
|
48
48
|
# - rdoc-ref:MetalArchives::Errors::InvalidIDError when no or invalid id
|
49
49
|
# - rdoc-ref:MetalArchives::Errors::APIError when receiving a status code >= 400 (except 404)
|
50
50
|
#
|
51
|
-
property :date_released, type:
|
51
|
+
property :date_released, type: Date
|
52
52
|
|
53
53
|
##
|
54
54
|
# :attr_reader_: catalog_id
|
@@ -124,12 +124,9 @@ module MetalArchives
|
|
124
124
|
#
|
125
125
|
def assemble # :nodoc:
|
126
126
|
## Base attributes
|
127
|
-
|
128
|
-
response = HTTPClient.get url
|
127
|
+
response = MetalArchives.http.get "/albums/view/id/#{id}"
|
129
128
|
|
130
|
-
|
131
|
-
|
132
|
-
properties
|
129
|
+
Parsers::Release.parse_html response.to_s
|
133
130
|
end
|
134
131
|
|
135
132
|
class << self
|
@@ -142,7 +139,7 @@ module MetalArchives
|
|
142
139
|
# +Integer+
|
143
140
|
#
|
144
141
|
def find(id)
|
145
|
-
return cache[id] if cache.include? id
|
142
|
+
return MetalArchives.cache[id] if MetalArchives.cache.include? id
|
146
143
|
|
147
144
|
Release.new id: id
|
148
145
|
end
|
@@ -200,11 +197,10 @@ module MetalArchives
|
|
200
197
|
# - +:formats+: +Array+ of +Symbol+, see rdoc-ref:Release.format
|
201
198
|
#
|
202
199
|
def find_by(query)
|
203
|
-
url = "#{MetalArchives.config.default_endpoint}search/ajax-advanced/searching/albums"
|
204
200
|
params = Parsers::Release.map_params query
|
205
201
|
|
206
|
-
response =
|
207
|
-
json = JSON.parse response.
|
202
|
+
response = MetalArchives.http.get "/search/ajax-advanced/searching/albums", params
|
203
|
+
json = JSON.parse response.to_s
|
208
204
|
|
209
205
|
return nil if json["aaData"].empty?
|
210
206
|
|
@@ -286,8 +282,6 @@ module MetalArchives
|
|
286
282
|
# - +:formats+: +Array+ of +Symbol+, see rdoc-ref:Release.format
|
287
283
|
#
|
288
284
|
def search_by(query)
|
289
|
-
url = "#{MetalArchives.config.default_endpoint}search/ajax-advanced/searching/albums"
|
290
|
-
|
291
285
|
params = Parsers::Release.map_params query
|
292
286
|
|
293
287
|
l = lambda do
|
@@ -296,8 +290,8 @@ module MetalArchives
|
|
296
290
|
if @max_items && @start >= @max_items
|
297
291
|
[]
|
298
292
|
else
|
299
|
-
response =
|
300
|
-
json = JSON.parse response.
|
293
|
+
response = MetalArchives.http.get "/search/ajax-advanced/searching/albums", params.merge(iDisplayStart: @start)
|
294
|
+
json = JSON.parse response.to_s
|
301
295
|
|
302
296
|
@max_items = json["iTotalRecords"]
|
303
297
|
|
@@ -20,11 +20,9 @@ module MetalArchives
|
|
20
20
|
# +Hash+
|
21
21
|
#
|
22
22
|
def map_params(query)
|
23
|
-
|
23
|
+
{
|
24
24
|
query: query[:name] || "",
|
25
25
|
}
|
26
|
-
|
27
|
-
params
|
28
26
|
end
|
29
27
|
|
30
28
|
##
|
@@ -36,13 +34,30 @@ module MetalArchives
|
|
36
34
|
# - rdoc-ref:MetalArchives::Errors::ParserError when parsing failed. Please report this error.
|
37
35
|
#
|
38
36
|
def parse_html(response)
|
39
|
-
|
37
|
+
# Set default props
|
38
|
+
props = {
|
39
|
+
name: nil,
|
40
|
+
aliases: [],
|
41
|
+
|
42
|
+
date_of_birth: nil,
|
43
|
+
date_of_death: nil,
|
44
|
+
cause_of_death: nil,
|
45
|
+
gender: nil,
|
46
|
+
|
47
|
+
country: nil,
|
48
|
+
location: nil,
|
49
|
+
|
50
|
+
photo: nil,
|
51
|
+
|
52
|
+
bands: [],
|
53
|
+
}
|
54
|
+
|
40
55
|
doc = Nokogiri::HTML response
|
41
56
|
|
42
57
|
# Photo
|
43
58
|
unless doc.css(".member_img").empty?
|
44
59
|
photo_uri = URI doc.css(".member_img img").first.attr("src")
|
45
|
-
props[:photo] =
|
60
|
+
props[:photo] = rewrite(photo_uri)
|
46
61
|
end
|
47
62
|
|
48
63
|
doc.css("#member_info dl").each do |dl|
|
@@ -55,25 +70,14 @@ module MetalArchives
|
|
55
70
|
when "Real/full name:"
|
56
71
|
props[:name] = content
|
57
72
|
when "Age:"
|
58
|
-
|
59
|
-
begin
|
60
|
-
props[:date_of_birth] = NilDate.parse date
|
61
|
-
rescue MetalArchives::Errors::ArgumentError => e
|
62
|
-
dob = Date.parse date
|
63
|
-
props[:date_of_birth] = NilDate.new dob.year, dob.month, dob.day
|
64
|
-
end
|
73
|
+
props[:date_of_birth] = Parsers::Date.parse(content.strip.gsub(/[0-9]* *\(born ([^)]*)\)/, '\1'))
|
65
74
|
when "R.I.P.:"
|
66
|
-
|
67
|
-
dod = Date.parse content
|
68
|
-
props[:date_of_death] = NilDate.new dod.year, dod.month, dod.day
|
69
|
-
rescue ArgumentError => e
|
70
|
-
props[:date_of_death] = NilDate.parse content
|
71
|
-
end
|
75
|
+
props[:date_of_death] = Parsers::Date.parse(content)
|
72
76
|
when "Died of:"
|
73
77
|
props[:cause_of_death] = content
|
74
78
|
when "Place of origin:"
|
75
|
-
props[:country] =
|
76
|
-
location = dt.next_element.xpath("text()").map(&:content).join
|
79
|
+
props[:country] = Country.parse(sanitize(dt.next_element.css("a").first.content))
|
80
|
+
location = dt.next_element.xpath("text()").map(&:content).join.strip.gsub(/[()]/, "")
|
77
81
|
props[:location] = location unless location.empty?
|
78
82
|
when "Gender:"
|
79
83
|
case content
|
@@ -91,33 +95,34 @@ module MetalArchives
|
|
91
95
|
end
|
92
96
|
|
93
97
|
# Aliases
|
94
|
-
props[:aliases] = []
|
95
98
|
alt = sanitize doc.css(".band_member_name").first.content
|
96
99
|
props[:aliases] << alt unless props[:name] == alt
|
97
100
|
|
98
101
|
# Active bands
|
99
|
-
props[:bands] = []
|
100
|
-
|
101
102
|
proc = proc do |row|
|
102
103
|
link = row.css("h3 a")
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
104
|
+
|
105
|
+
name, id = nil
|
106
|
+
|
107
|
+
if link.any?
|
108
|
+
# Band name contains a link
|
109
|
+
id = Integer(link.attr("href").text.gsub(%r(^.*/([^/#]*)#.*$), '\1'))
|
110
|
+
else
|
111
|
+
# Band name does not contain a link
|
112
|
+
name = sanitize row.css("h3").text
|
113
|
+
end
|
110
114
|
|
111
115
|
r = row.css(".member_in_band_role")
|
112
116
|
|
113
|
-
range =
|
117
|
+
range = Parsers::Year.parse(r.xpath("text()").map(&:content).join.strip.gsub(/[\n\r\t]/, "").gsub(/.*\((.*)\)/, '\1'))
|
114
118
|
role = sanitize r.css("strong").first.content
|
115
119
|
|
116
120
|
{
|
117
|
-
|
118
|
-
|
121
|
+
id: id,
|
122
|
+
name: name,
|
123
|
+
years_active: range,
|
119
124
|
role: role,
|
120
|
-
}
|
125
|
+
}.compact
|
121
126
|
end
|
122
127
|
|
123
128
|
doc.css("#artist_tab_active .member_in_band").each do |row|
|
@@ -151,7 +156,7 @@ module MetalArchives
|
|
151
156
|
type = :official
|
152
157
|
|
153
158
|
doc.css("#linksTablemain tr").each do |row|
|
154
|
-
if row["id"]
|
159
|
+
if /^header_/.match?(row["id"])
|
155
160
|
type = row["id"].gsub(/^header_/, "").downcase.to_sym
|
156
161
|
else
|
157
162
|
a = row.css("td a").first
|
@@ -24,8 +24,8 @@ module MetalArchives
|
|
24
24
|
bandName: query[:name] || "",
|
25
25
|
exactBandMatch: (query[:exact] ? 1 : 0),
|
26
26
|
genre: query[:genre] || "",
|
27
|
-
yearCreationFrom:
|
28
|
-
yearCreationTo:
|
27
|
+
yearCreationFrom: query[:year]&.begin || "",
|
28
|
+
yearCreationTo: query[:year]&.end || "",
|
29
29
|
bandNotes: query[:comment] || "",
|
30
30
|
status: map_status(query[:status]),
|
31
31
|
themes: query[:lyrical_themes] || "",
|
@@ -52,23 +52,42 @@ module MetalArchives
|
|
52
52
|
# - rdoc-ref:MetalArchives::Errors::ParserError when parsing failed. Please report this error.
|
53
53
|
#
|
54
54
|
def parse_html(response)
|
55
|
-
|
55
|
+
# Set default props
|
56
|
+
props = {
|
57
|
+
name: nil,
|
58
|
+
aliases: [],
|
59
|
+
|
60
|
+
logo: nil,
|
61
|
+
photo: nil,
|
62
|
+
|
63
|
+
country: nil,
|
64
|
+
location: nil,
|
65
|
+
|
66
|
+
status: nil,
|
67
|
+
date_formed: nil,
|
68
|
+
years_active: [],
|
69
|
+
independent: nil,
|
70
|
+
|
71
|
+
genres: [],
|
72
|
+
lyrical_themes: [],
|
73
|
+
|
74
|
+
members: [],
|
75
|
+
}
|
76
|
+
|
56
77
|
doc = Nokogiri::HTML response
|
57
78
|
|
58
79
|
props[:name] = sanitize doc.css("#band_info .band_name a").first.content
|
59
80
|
|
60
|
-
props[:aliases] = []
|
61
|
-
|
62
81
|
# Logo
|
63
82
|
unless doc.css(".band_name_img").empty?
|
64
83
|
logo_uri = URI doc.css(".band_name_img img").first.attr("src")
|
65
|
-
props[:logo] =
|
84
|
+
props[:logo] = rewrite(logo_uri)
|
66
85
|
end
|
67
86
|
|
68
87
|
# Photo
|
69
88
|
unless doc.css(".band_img").empty?
|
70
89
|
photo_uri = URI doc.css(".band_img img").first.attr("src")
|
71
|
-
props[:photo] =
|
90
|
+
props[:photo] = rewrite(photo_uri)
|
72
91
|
end
|
73
92
|
|
74
93
|
doc.css("#band_stats dl").each do |dl|
|
@@ -79,22 +98,16 @@ module MetalArchives
|
|
79
98
|
|
80
99
|
case dt.content
|
81
100
|
when "Country of origin:"
|
82
|
-
props[:country] =
|
101
|
+
props[:country] = Country.parse(sanitize(dt.next_element.css("a").first.content))
|
83
102
|
when "Location:"
|
84
103
|
props[:location] = content
|
85
104
|
when "Status:"
|
86
|
-
props[:status] = content.downcase.tr(" ", "_").to_sym
|
105
|
+
props[:status] = content.downcase.tr(" -", "_").to_sym
|
87
106
|
when "Formed in:"
|
88
|
-
|
89
|
-
dof = Date.parse content
|
90
|
-
props[:date_formed] = NilDate.new dof.year, dof.month, dof.day
|
91
|
-
rescue ArgumentError => e
|
92
|
-
props[:date_formed] = NilDate.parse content
|
93
|
-
end
|
107
|
+
props[:date_formed] = Parsers::Date.parse(content)
|
94
108
|
when "Genre:"
|
95
|
-
props[:genres] =
|
109
|
+
props[:genres] = Parsers::Genre.parse(content)
|
96
110
|
when "Lyrical themes:"
|
97
|
-
props[:lyrical_themes] = []
|
98
111
|
content.split(",").each do |theme|
|
99
112
|
t = theme.split.map(&:capitalize)
|
100
113
|
t.delete "(early)"
|
@@ -105,26 +118,56 @@ module MetalArchives
|
|
105
118
|
props[:independent] = (content == "Unsigned/independent")
|
106
119
|
# TODO: label
|
107
120
|
when "Years active:"
|
108
|
-
props[:date_active] = []
|
109
121
|
content.split(",").each do |range|
|
110
122
|
# Aliases
|
111
123
|
range.scan(/\(as ([^)]*)\)/).each { |name| props[:aliases] << name.first }
|
112
124
|
# Ranges
|
113
|
-
|
114
|
-
date_start = (r.first == "?" ? nil : NilDate.new(r.first.to_i))
|
115
|
-
date_end = (r.last == "?" || r.last == "present" ? nil : NilDate.new(r.first.to_i))
|
116
|
-
props[:date_active] << MetalArchives::Range.new(date_start, date_end)
|
125
|
+
props[:years_active] << Parsers::Year.parse(range.gsub(/ *\(as ([^)]*)\) */, ""))
|
117
126
|
end
|
118
127
|
else
|
119
|
-
raise
|
128
|
+
raise Errors::ParserError, "Unknown token: #{dt.content}"
|
120
129
|
end
|
121
130
|
end
|
122
131
|
end
|
123
132
|
|
133
|
+
# Members
|
134
|
+
proc = proc do |row|
|
135
|
+
link = row.css("a")
|
136
|
+
|
137
|
+
if link.any?
|
138
|
+
# Artist name contains a link
|
139
|
+
id = Integer(link.attr("href").text.split("/").last)
|
140
|
+
name = sanitize link.text
|
141
|
+
else
|
142
|
+
# Artist name does not contain a link
|
143
|
+
name = sanitize row.css("h3").text
|
144
|
+
end
|
145
|
+
|
146
|
+
r = row.css("td").last.text
|
147
|
+
role, range = r.match(/(.*)\(([^(]*)\)/).captures
|
148
|
+
|
149
|
+
range = Parsers::Year.parse(range)
|
150
|
+
|
151
|
+
{
|
152
|
+
id: id,
|
153
|
+
name: name,
|
154
|
+
years_active: range,
|
155
|
+
role: sanitize(role),
|
156
|
+
}.compact
|
157
|
+
end
|
158
|
+
|
159
|
+
doc.css("#band_tab_members_current .lineupRow").each do |row|
|
160
|
+
props[:members] << proc.call(row).merge(current: true)
|
161
|
+
end
|
162
|
+
|
163
|
+
doc.css("#band_tab_members_past .lineupRow").each do |row|
|
164
|
+
props[:members] << proc.call(row).merge(current: false)
|
165
|
+
end
|
166
|
+
|
124
167
|
props
|
125
168
|
rescue StandardError => e
|
126
169
|
e.backtrace.each { |b| MetalArchives.config.logger.error b }
|
127
|
-
raise
|
170
|
+
raise Errors::ParserError, e
|
128
171
|
end
|
129
172
|
|
130
173
|
##
|
@@ -141,15 +184,16 @@ module MetalArchives
|
|
141
184
|
doc = Nokogiri::HTML response
|
142
185
|
doc.css("#artist_list tbody tr").each do |row|
|
143
186
|
similar << {
|
144
|
-
|
187
|
+
id: row.css("td a").first["href"].split("/").last.to_i,
|
145
188
|
score: row.css("td").last.content.strip,
|
146
189
|
}
|
147
190
|
end
|
148
191
|
|
149
192
|
similar
|
150
193
|
rescue StandardError => e
|
194
|
+
MetalArchives.config.logger e.message
|
151
195
|
e.backtrace.each { |b| MetalArchives.config.logger.error b }
|
152
|
-
raise
|
196
|
+
raise Errors::ParserError, e
|
153
197
|
end
|
154
198
|
|
155
199
|
##
|
@@ -182,7 +226,7 @@ module MetalArchives
|
|
182
226
|
links
|
183
227
|
rescue StandardError => e
|
184
228
|
e.backtrace.each { |b| MetalArchives.config.logger.error b }
|
185
|
-
raise
|
229
|
+
raise Errors::ParserError, e
|
186
230
|
end
|
187
231
|
|
188
232
|
##
|
@@ -204,7 +248,7 @@ module MetalArchives
|
|
204
248
|
releases
|
205
249
|
rescue StandardError => e
|
206
250
|
e.backtrace.each { |b| MetalArchives.config.logger.error b }
|
207
|
-
raise
|
251
|
+
raise Errors::ParserError, e
|
208
252
|
end
|
209
253
|
|
210
254
|
private
|
@@ -228,7 +272,7 @@ module MetalArchives
|
|
228
272
|
:disputed => "Disputed",
|
229
273
|
}
|
230
274
|
|
231
|
-
raise
|
275
|
+
raise Errors::ParserError, "Unknown status: #{status}" unless s[status]
|
232
276
|
|
233
277
|
s[status]
|
234
278
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "countries"
|
4
|
+
|
5
|
+
module MetalArchives
|
6
|
+
module Parsers
|
7
|
+
##
|
8
|
+
# Country parser
|
9
|
+
#
|
10
|
+
class Country < Base
|
11
|
+
##
|
12
|
+
# Parse a country
|
13
|
+
#
|
14
|
+
# Returns +ISO3166::Country+
|
15
|
+
#
|
16
|
+
def self.parse(input)
|
17
|
+
ISO3166::Country.find_country_by_name(input)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|