bandcamp-discover 0.7.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.rdoc +4 -0
- data/lib/bandcamp-discover/scrapers/album.rb +3 -3
- data/lib/bandcamp-discover/scrapers/base.rb +13 -0
- data/lib/bandcamp-discover/scrapers/discover.rb +13 -4
- data/lib/bandcamp-discover/scrapers/label.rb +1 -1
- data/lib/bandcamp-discover/scrapers/music.rb +29 -8
- data/lib/bandcamp-discover/version.rb +1 -1
- data/test/discover_test.rb +12 -4
- data/test/music_test.rb +40 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ecd43fd0b46fd8157743d881e513bb9a34abbf6241819a48d60f8bc1e82ca084
|
|
4
|
+
data.tar.gz: 3cd9fed33d330e6db6231aa8d19988405c264bc458eef6e4fd86bd3a0b1bcad1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 86d35206ccb8681009ab5c46dd7fef88db2122e0e0763ad99c35eef3e8ec73ca73779a77351250536b3774fe0bf80a22fa75c1290ea3ff132309e60cb2861672
|
|
7
|
+
data.tar.gz: a6b8bc9bfe4ea77ecbe29db09da5baa2236aa580211b2bed4e8a61a20a105eb14a01a760ca4e1e4484b8563c2864069619b29e5f583969a72219537ab65b4bf0
|
data/Gemfile.lock
CHANGED
data/README.rdoc
CHANGED
|
@@ -21,6 +21,10 @@ bio for the model and returned as +artists+ in the scrape result; each
|
|
|
21
21
|
entry in +albums+ carries its own credit as +artist+ (nil when the release
|
|
22
22
|
is credited to the page owner).
|
|
23
23
|
|
|
24
|
+
Pages are read at DOMContentLoaded and images, media and fonts are never
|
|
25
|
+
fetched: everything the scrapers use is in the server-rendered HTML, and the
|
|
26
|
+
/music grid is completed from the JSON Bandcamp ships for its own script.
|
|
27
|
+
|
|
24
28
|
Without a token, +label?+ falls back to matching /label|platform|records/i
|
|
25
29
|
and +accepts_demos?+ is always false.
|
|
26
30
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
require_relative "
|
|
2
|
-
require_relative "
|
|
1
|
+
require_relative "base"
|
|
2
|
+
require_relative "music"
|
|
3
3
|
|
|
4
4
|
module BandcampDiscover
|
|
5
5
|
module Scrapers
|
|
6
6
|
class Album < Base
|
|
7
7
|
def scrape(force: false)
|
|
8
8
|
super do |page|
|
|
9
|
-
|
|
9
|
+
visit(@url)
|
|
10
10
|
|
|
11
11
|
title = page.query_selector("meta[name=title]")&.[](:content)
|
|
12
12
|
|
|
@@ -7,10 +7,16 @@ module BandcampDiscover
|
|
|
7
7
|
class ScrapeError < StandardError; end
|
|
8
8
|
|
|
9
9
|
class Base
|
|
10
|
+
# Nothing read from a page is an image, a player or a font, and the
|
|
11
|
+
# album pages are twenty per label; not fetching them is most of the
|
|
12
|
+
# time and bandwidth a scrape used to cost.
|
|
13
|
+
BLOCKED_RESOURCES = %w[image media font].freeze
|
|
14
|
+
|
|
10
15
|
def initialize(url:, browser:, max_tasks: 2)
|
|
11
16
|
@url = url
|
|
12
17
|
@browser = browser
|
|
13
18
|
@page = browser.new_page
|
|
19
|
+
@page.route("**/*", ->(route, request) { BLOCKED_RESOURCES.include?(request.resource_type) ? route.abort : route.continue })
|
|
14
20
|
@max_tasks = max_tasks
|
|
15
21
|
end
|
|
16
22
|
|
|
@@ -20,6 +26,13 @@ module BandcampDiscover
|
|
|
20
26
|
|
|
21
27
|
private
|
|
22
28
|
|
|
29
|
+
# Everything read is in the server-rendered HTML, so the DOM is enough;
|
|
30
|
+
# waiting for "load" waited on every tracker and player asset, and one
|
|
31
|
+
# slow one failed the whole label.
|
|
32
|
+
def visit(url)
|
|
33
|
+
@page.goto(url, waitUntil: "domcontentloaded")
|
|
34
|
+
end
|
|
35
|
+
|
|
23
36
|
def guarded
|
|
24
37
|
yield
|
|
25
38
|
rescue Playwright::TimeoutError => e
|
|
@@ -23,6 +23,11 @@ module BandcampDiscover
|
|
|
23
23
|
# "a" is albums, "s" is subscriptions.
|
|
24
24
|
RESULT_TYPES = %w[a s].freeze
|
|
25
25
|
|
|
26
|
+
# Bandcamp renamed the response fields from snake_case to camelCase in
|
|
27
|
+
# November 2026 while the request stayed snake_case. Both are read in case
|
|
28
|
+
# it rolls back or only part of their fleet has it.
|
|
29
|
+
BAND_URL_KEYS = %w[bandUrl band_url].freeze
|
|
30
|
+
|
|
26
31
|
class ResponseError < StandardError; end
|
|
27
32
|
|
|
28
33
|
# browser: is unused, kept so existing callers do not break.
|
|
@@ -69,7 +74,7 @@ module BandcampDiscover
|
|
|
69
74
|
results = body["results"]
|
|
70
75
|
break if results.empty?
|
|
71
76
|
|
|
72
|
-
urls.concat(results.map { root_url(_1
|
|
77
|
+
urls.concat(results.map { root_url(band_url(_1)) }.compact)
|
|
73
78
|
|
|
74
79
|
cursor = body["cursor"]
|
|
75
80
|
break if cursor.nil? || cursor.empty?
|
|
@@ -115,12 +120,16 @@ module BandcampDiscover
|
|
|
115
120
|
raise ResponseError, "discover_web response has no results array (keys: #{body.is_a?(Hash) ? body.keys.inspect : body.class})"
|
|
116
121
|
end
|
|
117
122
|
|
|
118
|
-
return if body["results"].empty? || body["results"].any? { _1.is_a?(Hash) && _1.
|
|
123
|
+
return if body["results"].empty? || body["results"].any? { _1.is_a?(Hash) && _1.keys.intersect?(BAND_URL_KEYS) }
|
|
124
|
+
|
|
125
|
+
raise ResponseError, "discover_web results carry no band url (looked for #{BAND_URL_KEYS.join(", ")}; keys: #{body["results"].first.keys.inspect})"
|
|
126
|
+
end
|
|
119
127
|
|
|
120
|
-
|
|
128
|
+
def band_url(result)
|
|
129
|
+
result.values_at(*BAND_URL_KEYS).compact.first
|
|
121
130
|
end
|
|
122
131
|
|
|
123
|
-
# Strips the ?from=discover_page
|
|
132
|
+
# Strips the ?from=discover_page the band url carries.
|
|
124
133
|
def root_url(band_url)
|
|
125
134
|
return nil if band_url.nil? || band_url.empty?
|
|
126
135
|
|
|
@@ -2,6 +2,7 @@ require_relative "base"
|
|
|
2
2
|
require_relative "album"
|
|
3
3
|
require_relative "../roster"
|
|
4
4
|
require "async"
|
|
5
|
+
require "json"
|
|
5
6
|
require "async/semaphore"
|
|
6
7
|
|
|
7
8
|
module BandcampDiscover
|
|
@@ -23,18 +24,33 @@ module BandcampDiscover
|
|
|
23
24
|
# The grid alone tells a label from an artist (see Roster), and it is one
|
|
24
25
|
# page load against the twenty behind albums, so callers can look at it
|
|
25
26
|
# before paying for the rest.
|
|
27
|
+
#
|
|
28
|
+
# Bandcamp renders the first sixteen items as HTML and ships the rest as
|
|
29
|
+
# JSON in data-client-items for its own script to render after load. Read
|
|
30
|
+
# both and no script has to run: the grid is complete at DOMContentLoaded.
|
|
26
31
|
def grid
|
|
27
32
|
guarded do
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
{
|
|
33
|
-
url: absolute(item.query_selector("a")[:href]),
|
|
34
|
-
credit: item.query_selector(".artist-override")&.inner_text&.strip
|
|
35
|
-
}
|
|
33
|
+
visit(@url)
|
|
34
|
+
grid = @page.wait_for_selector("#music-grid")
|
|
35
|
+
|
|
36
|
+
rendered = grid.query_selector_all("li.music-grid-item").map do |item|
|
|
37
|
+
{url: item.query_selector("a")[:href], credit: item.query_selector(".artist-override")&.inner_text}
|
|
36
38
|
end
|
|
39
|
+
|
|
40
|
+
merge_grid(rendered, grid.get_attribute("data-client-items"))
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# A credit is present in the JSON only when the release is not the
|
|
45
|
+
# owner's; the same convention as the rendered override.
|
|
46
|
+
def merge_grid(rendered, client_items_json)
|
|
47
|
+
client = JSON.parse(client_items_json.to_s.empty? ? "[]" : client_items_json).map do |item|
|
|
48
|
+
{url: item["page_url"], credit: item["artist"]}
|
|
37
49
|
end
|
|
50
|
+
|
|
51
|
+
(rendered + client)
|
|
52
|
+
.map { |item| {url: absolute(item[:url]), credit: credit_or_nil(item[:credit])} }
|
|
53
|
+
.uniq { |item| item[:url] }
|
|
38
54
|
end
|
|
39
55
|
|
|
40
56
|
def roster(grid, band_name:)
|
|
@@ -79,6 +95,11 @@ module BandcampDiscover
|
|
|
79
95
|
def absolute(href)
|
|
80
96
|
href.start_with?("https://") ? href : "#{@base_url}#{href}"
|
|
81
97
|
end
|
|
98
|
+
|
|
99
|
+
def credit_or_nil(credit)
|
|
100
|
+
text = credit.to_s.strip
|
|
101
|
+
text.empty? ? nil : text
|
|
102
|
+
end
|
|
82
103
|
end
|
|
83
104
|
end
|
|
84
105
|
end
|
data/test/discover_test.rb
CHANGED
|
@@ -14,7 +14,7 @@ class DiscoverTest < Minitest::Test
|
|
|
14
14
|
|
|
15
15
|
def batch(band_urls, cursor: nil)
|
|
16
16
|
{
|
|
17
|
-
"results" => band_urls.map { {"
|
|
17
|
+
"results" => band_urls.map { {"bandUrl" => _1, "itemType" => "a"} },
|
|
18
18
|
"cursor" => cursor
|
|
19
19
|
}
|
|
20
20
|
end
|
|
@@ -71,11 +71,19 @@ class DiscoverTest < Minitest::Test
|
|
|
71
71
|
assert_match(/no results array/, error.message)
|
|
72
72
|
end
|
|
73
73
|
|
|
74
|
-
def
|
|
75
|
-
scraper = build(batches: [{"results" => [{"
|
|
74
|
+
def test_raises_when_results_no_longer_carry_a_band_url
|
|
75
|
+
scraper = build(batches: [{"results" => [{"itemId" => 1, "bandName" => "x"}], "cursor" => nil}])
|
|
76
76
|
|
|
77
77
|
error = assert_raises(Discover::ResponseError) { collect(scraper) }
|
|
78
|
-
assert_match(/no
|
|
78
|
+
assert_match(/no band url/, error.message)
|
|
79
|
+
assert_match(/itemId/, error.message)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# The spelling the API used until November 2026.
|
|
83
|
+
def test_still_reads_snake_case_results
|
|
84
|
+
scraper = build(batches: [{"results" => [{"band_url" => "https://a.bandcamp.com?from=discover_page"}], "cursor" => nil}])
|
|
85
|
+
|
|
86
|
+
assert_equal ["https://a.bandcamp.com"], collect(scraper)
|
|
79
87
|
end
|
|
80
88
|
|
|
81
89
|
def test_empty_results_is_not_an_error
|
data/test/music_test.rb
CHANGED
|
@@ -6,8 +6,12 @@ require "bandcamp-discover/scrapers/music"
|
|
|
6
6
|
class MusicTest < Minitest::Test
|
|
7
7
|
Music = BandcampDiscover::Scrapers::Music
|
|
8
8
|
|
|
9
|
+
FakePage = Struct.new(:routes) do
|
|
10
|
+
def route(pattern, handler) = (self.routes ||= []) << pattern
|
|
11
|
+
end
|
|
12
|
+
|
|
9
13
|
FakeBrowser = Struct.new(:pages) do
|
|
10
|
-
def new_page =
|
|
14
|
+
def new_page = FakePage.new
|
|
11
15
|
end
|
|
12
16
|
|
|
13
17
|
FakeAlbum = Struct.new(:url) do
|
|
@@ -43,4 +47,39 @@ class MusicTest < Minitest::Test
|
|
|
43
47
|
|
|
44
48
|
assert_equal [{url: nil, title: nil, tags: nil, player_url: nil, artist: "Helen"}], albums
|
|
45
49
|
end
|
|
50
|
+
|
|
51
|
+
# Bandcamp renders sixteen items and ships the rest as JSON for its own
|
|
52
|
+
# script; reading both makes the grid complete without running it.
|
|
53
|
+
def test_grid_merges_rendered_items_with_the_client_items_json
|
|
54
|
+
rendered = [
|
|
55
|
+
{url: "/album/overflowing", credit: " Early Moon "},
|
|
56
|
+
{url: "/album/own-release", credit: nil}
|
|
57
|
+
]
|
|
58
|
+
json = '[{"art_id":1,"artist":"David Cordero","band_id":3141249859,"id":2,"page_url":"/album/and-stillness-came","title":"And Stillness Came","type":"album"},' \
|
|
59
|
+
'{"art_id":3,"band_id":3141249859,"id":4,"page_url":"/track/pay","title":"Pay","type":"track"},' \
|
|
60
|
+
'{"art_id":5,"artist":"","band_id":3141249859,"id":6,"page_url":"/album/overflowing","title":"dup","type":"album"}]'
|
|
61
|
+
|
|
62
|
+
grid = @music.merge_grid(rendered, json)
|
|
63
|
+
|
|
64
|
+
assert_equal [
|
|
65
|
+
{url: "https://kranky.bandcamp.com/album/overflowing", credit: "Early Moon"},
|
|
66
|
+
{url: "https://kranky.bandcamp.com/album/own-release", credit: nil},
|
|
67
|
+
{url: "https://kranky.bandcamp.com/album/and-stillness-came", credit: "David Cordero"},
|
|
68
|
+
{url: "https://kranky.bandcamp.com/track/pay", credit: nil}
|
|
69
|
+
], grid
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def test_grid_without_client_items_is_the_rendered_items
|
|
73
|
+
rendered = [{url: "https://radiohead.bandcamp.com/album/kid-a-mnesia", credit: nil}]
|
|
74
|
+
|
|
75
|
+
assert_equal rendered, @music.merge_grid(rendered, nil)
|
|
76
|
+
assert_equal rendered, @music.merge_grid(rendered, "")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def test_pages_block_images_media_and_fonts
|
|
80
|
+
page = @music.instance_variable_get(:@page)
|
|
81
|
+
|
|
82
|
+
assert_equal ["**/*"], page.routes
|
|
83
|
+
assert_equal %w[image media font], BandcampDiscover::Scrapers::Base::BLOCKED_RESOURCES
|
|
84
|
+
end
|
|
46
85
|
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bandcamp-discover
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Julian RUbisch
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2026-
|
|
10
|
+
date: 2026-11-19 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: rake
|