metal_archives 2.2.3 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +59 -12
- data/.rspec +1 -0
- data/.rubocop.yml +34 -20
- data/CHANGELOG.md +16 -1
- data/LICENSE.md +17 -4
- data/README.md +37 -29
- data/bin/console +8 -11
- data/config/inflections.rb +7 -0
- data/config/initializers/.keep +0 -0
- data/docker-compose.yml +10 -1
- data/lib/metal_archives.rb +57 -21
- data/lib/metal_archives/cache/base.rb +40 -0
- data/lib/metal_archives/cache/memory.rb +68 -0
- data/lib/metal_archives/cache/null.rb +22 -0
- data/lib/metal_archives/cache/redis.rb +49 -0
- data/lib/metal_archives/collection.rb +3 -5
- data/lib/metal_archives/configuration.rb +28 -21
- data/lib/metal_archives/errors.rb +9 -1
- data/lib/metal_archives/http_client.rb +42 -46
- data/lib/metal_archives/models/artist.rb +55 -26
- data/lib/metal_archives/models/band.rb +43 -36
- data/lib/metal_archives/models/{base_model.rb → base.rb} +57 -50
- data/lib/metal_archives/models/label.rb +7 -8
- data/lib/metal_archives/models/release.rb +21 -18
- data/lib/metal_archives/parsers/artist.rb +41 -36
- data/lib/metal_archives/parsers/band.rb +73 -29
- data/lib/metal_archives/parsers/base.rb +14 -0
- data/lib/metal_archives/parsers/country.rb +21 -0
- data/lib/metal_archives/parsers/date.rb +31 -0
- data/lib/metal_archives/parsers/genre.rb +67 -0
- data/lib/metal_archives/parsers/label.rb +21 -13
- data/lib/metal_archives/parsers/parser.rb +17 -77
- data/lib/metal_archives/parsers/release.rb +29 -18
- data/lib/metal_archives/parsers/year.rb +31 -0
- data/lib/metal_archives/version.rb +3 -3
- data/metal_archives.env.example +7 -4
- data/metal_archives.gemspec +7 -4
- data/nginx/default.conf +2 -2
- metadata +76 -32
- data/.github/workflows/release.yml +0 -69
- data/.rubocop_todo.yml +0 -92
- data/lib/metal_archives/lru_cache.rb +0 -61
- data/lib/metal_archives/middleware/cache_check.rb +0 -18
- data/lib/metal_archives/middleware/encoding.rb +0 -16
- data/lib/metal_archives/middleware/headers.rb +0 -38
- data/lib/metal_archives/middleware/rewrite_endpoint.rb +0 -38
- data/lib/metal_archives/nil_date.rb +0 -91
- data/lib/metal_archives/range.rb +0 -69
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "countries"
|
4
|
+
|
5
|
+
module MetalArchives
|
6
|
+
module Parsers
|
7
|
+
##
|
8
|
+
# Country parser
|
9
|
+
#
|
10
|
+
class Country < Base
|
11
|
+
##
|
12
|
+
# Parse a country
|
13
|
+
#
|
14
|
+
# Returns +ISO3166::Country+
|
15
|
+
#
|
16
|
+
def self.parse(input)
|
17
|
+
ISO3166::Country.find_country_by_name(input)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MetalArchives
|
4
|
+
module Parsers
|
5
|
+
##
|
6
|
+
# Date parser
|
7
|
+
#
|
8
|
+
class Date < Base
|
9
|
+
##
|
10
|
+
# Parse a date
|
11
|
+
#
|
12
|
+
# Returns +Date+
|
13
|
+
#
|
14
|
+
def self.parse(input)
|
15
|
+
::Date.parse(input)
|
16
|
+
rescue ::Date::Error
|
17
|
+
components = input
|
18
|
+
.split("-")
|
19
|
+
.map(&:to_i)
|
20
|
+
.reject(&:zero?)
|
21
|
+
.compact
|
22
|
+
|
23
|
+
return if components.empty?
|
24
|
+
|
25
|
+
::Date.new(*components)
|
26
|
+
rescue TypeError
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MetalArchives
|
4
|
+
module Parsers
|
5
|
+
##
|
6
|
+
# Genre parser
|
7
|
+
#
|
8
|
+
class Genre < Base
|
9
|
+
SUFFIXES = %w((early) (later) metal).freeze
|
10
|
+
|
11
|
+
##
|
12
|
+
# Opinionated parsing of genres
|
13
|
+
#
|
14
|
+
# Returns an +Array+ of +String+
|
15
|
+
#
|
16
|
+
# The following components are omitted:
|
17
|
+
# - Metal
|
18
|
+
# - (early)
|
19
|
+
# - (later)
|
20
|
+
#
|
21
|
+
# All genres are capitalized.
|
22
|
+
#
|
23
|
+
# For examples on how genres are parsed, refer to +gnre_spec.rb+
|
24
|
+
#
|
25
|
+
def self.parse(input)
|
26
|
+
genres = []
|
27
|
+
# Split fields
|
28
|
+
input.split(/[,;]/).each do |genre|
|
29
|
+
##
|
30
|
+
# Start with a single empty genre string. Split the genre by spaces
|
31
|
+
# and process each component. If a component does not have a slash,
|
32
|
+
# concatenate it to all genre strings present in +temp+. If it does
|
33
|
+
# have a slash present, duplicate all genre strings, and concatenate
|
34
|
+
# the first component (before the slash) to the first half, and the
|
35
|
+
# last component to the last half. +temp+ now has an array of genre
|
36
|
+
# combinations.
|
37
|
+
#
|
38
|
+
# 'Traditional Heavy/Power Metal' => ['Traditional Heavy', 'Traditional Power']
|
39
|
+
# 'Traditional/Classical Heavy/Power Metal' => [
|
40
|
+
# 'Traditional Heavy', 'Traditional Power',
|
41
|
+
# 'Classical Heavy', 'Classical Power']
|
42
|
+
#
|
43
|
+
temp = [""]
|
44
|
+
|
45
|
+
genre.downcase.split.reject { |g| SUFFIXES.include? g }.each do |g|
|
46
|
+
if g.include? "/"
|
47
|
+
# Duplicate all WIP genres
|
48
|
+
temp2 = temp.dup
|
49
|
+
|
50
|
+
# Assign first and last components to temp and temp2 respectively
|
51
|
+
split = g.split "/"
|
52
|
+
temp.map! { |t| t.empty? ? split.first.capitalize : "#{t.capitalize} #{split.first.capitalize}" }
|
53
|
+
temp2.map! { |t| t.empty? ? split.last.capitalize : "#{t.capitalize} #{split.last.capitalize}" }
|
54
|
+
|
55
|
+
# Add both genre trees
|
56
|
+
temp += temp2
|
57
|
+
else
|
58
|
+
temp.map! { |t| t.empty? ? g.capitalize : "#{t.capitalize} #{g.capitalize}" }
|
59
|
+
end
|
60
|
+
end
|
61
|
+
genres += temp
|
62
|
+
end
|
63
|
+
genres.uniq
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -11,16 +11,28 @@ module MetalArchives
|
|
11
11
|
class Label # :nodoc:
|
12
12
|
class << self
|
13
13
|
def find_endpoint(params)
|
14
|
-
"#{MetalArchives.config.
|
14
|
+
"#{MetalArchives.config.endpoint}labels/#{params[:name]}/#{params[:id]}"
|
15
15
|
end
|
16
16
|
|
17
17
|
def parse(response)
|
18
|
-
|
18
|
+
# Set default props
|
19
|
+
props = {
|
20
|
+
name: nil,
|
21
|
+
contact: [],
|
22
|
+
address: nil,
|
23
|
+
country: nil,
|
24
|
+
phone: nil,
|
25
|
+
status: nil,
|
26
|
+
specialization: [],
|
27
|
+
date_founded: nil,
|
28
|
+
|
29
|
+
online_shopping: nil,
|
30
|
+
}
|
31
|
+
|
19
32
|
doc = Nokogiri::HTML(response)
|
20
33
|
|
21
34
|
props[:name] = doc.css("#label_info .label_name").first.content
|
22
35
|
|
23
|
-
props[:contact] = []
|
24
36
|
doc.css("#label_contact a").each do |contact|
|
25
37
|
props[:contact] << {
|
26
38
|
title: contact.content,
|
@@ -38,26 +50,22 @@ module MetalArchives
|
|
38
50
|
when "Address:"
|
39
51
|
props[:address] = content
|
40
52
|
when "Country:"
|
41
|
-
props[:country] =
|
53
|
+
props[:country] = Country.parse(css("a").first.content)
|
42
54
|
when "Phone number:"
|
43
55
|
props[:phone] = content
|
44
56
|
when "Status:"
|
45
57
|
props[:status] = content.downcase.tr(" ", "_").to_sym
|
46
58
|
when "Specialised in:"
|
47
|
-
props[:specializations] =
|
59
|
+
props[:specializations] = Parsers::Genre.parse(content)
|
48
60
|
when "Founding date :"
|
49
|
-
|
50
|
-
dof = Date.parse content
|
51
|
-
props[:date_founded] = NilDate.new dof.year, dof.month, dof.day
|
52
|
-
rescue ArgumentError => e
|
53
|
-
props[:date_founded] = NilDate.parse content
|
54
|
-
end
|
61
|
+
props[:date_founded] = Parsers::Date.parse(content)
|
55
62
|
when "Sub-labels:"
|
56
63
|
# TODO
|
57
64
|
when "Online shopping:"
|
58
|
-
|
65
|
+
case content
|
66
|
+
when "Yes"
|
59
67
|
props[:online_shopping] = true
|
60
|
-
|
68
|
+
when "No"
|
61
69
|
props[:online_shopping] = false
|
62
70
|
end
|
63
71
|
else
|
@@ -1,103 +1,43 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "date"
|
4
|
-
require "countries"
|
5
4
|
|
6
5
|
module MetalArchives
|
7
|
-
|
8
|
-
# Mapping layer from and to MA Web Service
|
9
|
-
#
|
10
|
-
module Parsers # :nodoc:
|
6
|
+
module Parsers
|
11
7
|
##
|
12
8
|
# Parser base class
|
13
9
|
#
|
14
10
|
class Parser
|
15
11
|
class << self
|
16
|
-
##
|
17
|
-
# Parse a country
|
18
|
-
#
|
19
|
-
# Returns +ISO3166::Country+
|
20
|
-
#
|
21
|
-
def parse_country(input)
|
22
|
-
ISO3166::Country.find_country_by_name input
|
23
|
-
end
|
24
|
-
|
25
12
|
##
|
26
13
|
# Sanitize a string
|
27
14
|
#
|
28
15
|
# Return +String+
|
29
16
|
#
|
30
17
|
def sanitize(input)
|
31
|
-
|
18
|
+
return if input.blank?
|
19
|
+
|
20
|
+
input
|
21
|
+
.gsub(/^"/, "")
|
22
|
+
.gsub(/"$/, "")
|
23
|
+
.gsub(/[[:space:]]/, " ")
|
24
|
+
.strip
|
32
25
|
end
|
33
26
|
|
34
27
|
##
|
35
|
-
#
|
36
|
-
#
|
37
|
-
# Returns an +Array+ of +String+
|
28
|
+
# Rewrite a URL
|
38
29
|
#
|
39
|
-
#
|
40
|
-
# - Metal
|
41
|
-
# - (early)
|
42
|
-
# - (later)
|
30
|
+
# Return +URI+
|
43
31
|
#
|
44
|
-
|
45
|
-
|
46
|
-
# For examples on how genres are parsed, refer to +ParserTest#test_parse_genre+
|
47
|
-
#
|
48
|
-
def parse_genre(input)
|
49
|
-
genres = []
|
50
|
-
# Split fields
|
51
|
-
input.split(",").each do |genre|
|
52
|
-
##
|
53
|
-
# Start with a single empty genre string. Split the genre by spaces
|
54
|
-
# and process each component. If a component does not have a slash,
|
55
|
-
# concatenate it to all genre strings present in +temp+. If it does
|
56
|
-
# have a slash present, duplicate all genre strings, and concatenate
|
57
|
-
# the first component (before the slash) to the first half, and the
|
58
|
-
# last component to the last half. +temp+ now has an array of genre
|
59
|
-
# combinations.
|
60
|
-
#
|
61
|
-
# 'Traditional Heavy/Power Metal' => ['Traditional Heavy', 'Traditional Power']
|
62
|
-
# 'Traditional/Classical Heavy/Power Metal' => [
|
63
|
-
# 'Traditional Heavy', 'Traditional Power',
|
64
|
-
# 'Classical Heavy', 'Classical Power']
|
65
|
-
#
|
66
|
-
temp = [""]
|
67
|
-
genre.downcase.split.reject { |g| ["(early)", "(later)", "metal"].include? g }.each do |g|
|
68
|
-
if g.include? "/"
|
69
|
-
# Duplicate all WIP genres
|
70
|
-
temp2 = temp.dup
|
32
|
+
def rewrite(input)
|
33
|
+
return input unless MetalArchives.config.endpoint
|
71
34
|
|
72
|
-
|
73
|
-
split = g.split "/"
|
74
|
-
temp.map! { |t| t.empty? ? split.first.capitalize : "#{t.capitalize} #{split.first.capitalize}" }
|
75
|
-
temp2.map! { |t| t.empty? ? split.last.capitalize : "#{t.capitalize} #{split.last.capitalize}" }
|
76
|
-
|
77
|
-
# Add both genre trees
|
78
|
-
temp += temp2
|
79
|
-
else
|
80
|
-
temp.map! { |t| t.empty? ? g.capitalize : "#{t.capitalize} #{g.capitalize}" }
|
81
|
-
end
|
82
|
-
end
|
83
|
-
genres += temp
|
84
|
-
end
|
85
|
-
genres.uniq
|
86
|
-
end
|
87
|
-
|
88
|
-
##
|
89
|
-
# Parse year range
|
90
|
-
#
|
91
|
-
def parse_year_range(input)
|
92
|
-
r = input.split("-")
|
93
|
-
date_start = (r.first == "?" ? nil : NilDate.new(r.first.to_i))
|
94
|
-
date_end = if r.length > 1
|
95
|
-
(r.last == "?" || r.last == "present" ? nil : NilDate.new(r.last.to_i))
|
96
|
-
else
|
97
|
-
date_start.dup
|
98
|
-
end
|
35
|
+
endpoint = URI(MetalArchives.config.endpoint)
|
99
36
|
|
100
|
-
|
37
|
+
URI(input)
|
38
|
+
.tap { |u| u.host = endpoint.host }
|
39
|
+
.tap { |u| u.scheme = endpoint.scheme }
|
40
|
+
.to_s
|
101
41
|
end
|
102
42
|
end
|
103
43
|
end
|
@@ -71,7 +71,7 @@ module MetalArchives
|
|
71
71
|
# +Hash+
|
72
72
|
#
|
73
73
|
def map_params(query)
|
74
|
-
|
74
|
+
{
|
75
75
|
bandName: query[:band_name] || "",
|
76
76
|
releaseTitle: query[:title] || "",
|
77
77
|
releaseYearFrom: query[:from_year] || "",
|
@@ -90,8 +90,6 @@ module MetalArchives
|
|
90
90
|
releaseType: map_types(query[:types]),
|
91
91
|
releaseFormat: map_formats(query[:formats]),
|
92
92
|
}
|
93
|
-
|
94
|
-
params
|
95
93
|
end
|
96
94
|
|
97
95
|
##
|
@@ -103,11 +101,29 @@ module MetalArchives
|
|
103
101
|
# - rdoc-ref:MetalArchives::Errors::ParserError when parsing failed. Please report this error.
|
104
102
|
#
|
105
103
|
def parse_html(response)
|
106
|
-
|
104
|
+
# Set default props
|
105
|
+
props = {
|
106
|
+
title: nil,
|
107
|
+
band: nil,
|
108
|
+
type: nil,
|
109
|
+
date_released: nil,
|
110
|
+
catalog_id: nil,
|
111
|
+
identifier: nil,
|
112
|
+
version_description: nil,
|
113
|
+
format: nil,
|
114
|
+
limitation: nil,
|
115
|
+
}
|
116
|
+
|
107
117
|
doc = Nokogiri::HTML response
|
108
118
|
|
109
119
|
props[:title] = sanitize doc.css("#album_info .album_name a").first.content
|
110
120
|
|
121
|
+
# Band
|
122
|
+
band_doc = doc.css("#album_info .band_name a").first
|
123
|
+
id = Integer(band_doc.attr("href").split("/").last)
|
124
|
+
|
125
|
+
props[:band] = MetalArchives::Band.find(id)
|
126
|
+
|
111
127
|
doc.css("#album_info dl").each do |dl|
|
112
128
|
dl.search("dt").each do |dt|
|
113
129
|
content = sanitize dt.next_element.content
|
@@ -118,12 +134,7 @@ module MetalArchives
|
|
118
134
|
when "Type:"
|
119
135
|
props[:type] = map_type content
|
120
136
|
when "Release date:"
|
121
|
-
|
122
|
-
props[:date_released] = NilDate.parse content
|
123
|
-
rescue MetalArchives::Errors::ArgumentError => e
|
124
|
-
dr = Date.parse content
|
125
|
-
props[:date_released] = NilDate.new dr.year, dr.month, dr.day
|
126
|
-
end
|
137
|
+
props[:date_released] = Parsers::Date.parse(content)
|
127
138
|
when "Catalog ID:"
|
128
139
|
props[:catalog_id] = content
|
129
140
|
when "Identifier:"
|
@@ -140,7 +151,7 @@ module MetalArchives
|
|
140
151
|
next if content == "None yet"
|
141
152
|
# TODO: reviews
|
142
153
|
else
|
143
|
-
raise
|
154
|
+
raise Errors::ParserError, "Unknown token: #{dt.content}"
|
144
155
|
end
|
145
156
|
end
|
146
157
|
end
|
@@ -178,7 +189,7 @@ module MetalArchives
|
|
178
189
|
|
179
190
|
types = []
|
180
191
|
type_syms.each do |type|
|
181
|
-
raise
|
192
|
+
raise Errors::ParserError, "Unknown type: #{type}" unless TYPE_TO_QUERY[type]
|
182
193
|
|
183
194
|
types << TYPE_TO_QUERY[type]
|
184
195
|
end
|
@@ -192,7 +203,7 @@ module MetalArchives
|
|
192
203
|
# Returns +Symbol+, see rdoc-ref:Release.type
|
193
204
|
#
|
194
205
|
def map_type(type)
|
195
|
-
raise
|
206
|
+
raise Errors::ParserError, "Unknown type: #{type}" unless TYPE_TO_SYM[type]
|
196
207
|
|
197
208
|
TYPE_TO_SYM[type]
|
198
209
|
end
|
@@ -210,7 +221,7 @@ module MetalArchives
|
|
210
221
|
|
211
222
|
formats = []
|
212
223
|
format_syms.each do |format|
|
213
|
-
raise
|
224
|
+
raise Errors::ParserError, "Unknown format: #{format}" unless FORMAT_TO_QUERY[format]
|
214
225
|
|
215
226
|
formats << FORMAT_TO_QUERY[format]
|
216
227
|
end
|
@@ -224,11 +235,11 @@ module MetalArchives
|
|
224
235
|
# Returns +Symbol+, see rdoc-ref:Release.format
|
225
236
|
#
|
226
237
|
def map_format(format)
|
227
|
-
return :cd if
|
228
|
-
return :vinyl if
|
229
|
-
return :blu_ray if
|
238
|
+
return :cd if /CD/.match?(format)
|
239
|
+
return :vinyl if /[Vv]inyl/.match?(format)
|
240
|
+
return :blu_ray if /[Bb]lu.?[Rr]ay/.match?(format)
|
230
241
|
|
231
|
-
raise
|
242
|
+
raise Errors::ParserError, "Unknown format: #{format}" unless FORMAT_TO_SYM[format]
|
232
243
|
|
233
244
|
FORMAT_TO_SYM[format]
|
234
245
|
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MetalArchives
|
4
|
+
module Parsers
|
5
|
+
##
|
6
|
+
# Year range parser
|
7
|
+
#
|
8
|
+
class Year < Base
|
9
|
+
##
|
10
|
+
# Parse year range
|
11
|
+
#
|
12
|
+
# Returns +Range+ of +Integer+
|
13
|
+
#
|
14
|
+
def self.parse(input)
|
15
|
+
return if input.blank?
|
16
|
+
|
17
|
+
components = input
|
18
|
+
.split("-")
|
19
|
+
.map(&:to_i)
|
20
|
+
.map { |y| y.zero? ? nil : y }
|
21
|
+
|
22
|
+
return if components.empty?
|
23
|
+
|
24
|
+
# Set end if only one year
|
25
|
+
components << components.first if components.count == 1
|
26
|
+
|
27
|
+
components[0]..components[1]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|