bandcamp-discover 0.7.2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '094dffa15caad4bbabef29bd4b3d87789425e4736c2d06461d6917efc8fe600d'
4
- data.tar.gz: de1a1d20425f1ecb984878b10541208849abbf7300cd46f80834e5197730ef1c
3
+ metadata.gz: ecd43fd0b46fd8157743d881e513bb9a34abbf6241819a48d60f8bc1e82ca084
4
+ data.tar.gz: 3cd9fed33d330e6db6231aa8d19988405c264bc458eef6e4fd86bd3a0b1bcad1
5
5
  SHA512:
6
- metadata.gz: a6a057d4869da1039b00b9c2ffd5b4019123c4ee055cf392fd3fefc4b8fa7f427d2346081554e960a5416c95b52d507d9244912112dada5d0ddedec06820ac33
7
- data.tar.gz: ae37374ebc23a193aedaed32cd741f01af048502cb9853596d6e9bf1438ec2040fcacdaee67f226951accf6bd9d674191faf8667f5bf86cc3b9216d84a408bbc
6
+ metadata.gz: 86d35206ccb8681009ab5c46dd7fef88db2122e0e0763ad99c35eef3e8ec73ca73779a77351250536b3774fe0bf80a22fa75c1290ea3ff132309e60cb2861672
7
+ data.tar.gz: a6b8bc9bfe4ea77ecbe29db09da5baa2236aa580211b2bed4e8a61a20a105eb14a01a760ca4e1e4484b8563c2864069619b29e5f583969a72219537ab65b4bf0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- bandcamp-discover (0.7.2)
4
+ bandcamp-discover (0.8.1)
5
5
  async
6
6
  base64
7
7
  concurrent-ruby
data/README.rdoc CHANGED
@@ -21,6 +21,10 @@ bio for the model and returned as +artists+ in the scrape result; each
21
21
  entry in +albums+ carries its own credit as +artist+ (nil when the release
22
22
  is credited to the page owner).
23
23
 
24
+ Pages are read at DOMContentLoaded and images, media and fonts are never
25
+ fetched: everything the scrapers use is in the server-rendered HTML, and the
26
+ /music grid is completed from the JSON Bandcamp ships for its own script.
27
+
24
28
  Without a token, +label?+ falls back to matching /label|platform|records/i
25
29
  and +accepts_demos?+ is always false.
26
30
 
@@ -1,12 +1,12 @@
1
- require_relative "./base"
2
- require_relative "./music"
1
+ require_relative "base"
2
+ require_relative "music"
3
3
 
4
4
  module BandcampDiscover
5
5
  module Scrapers
6
6
  class Album < Base
7
7
  def scrape(force: false)
8
8
  super do |page|
9
- page.goto(@url)
9
+ visit(@url)
10
10
 
11
11
  title = page.query_selector("meta[name=title]")&.[](:content)
12
12
 
@@ -7,10 +7,16 @@ module BandcampDiscover
7
7
  class ScrapeError < StandardError; end
8
8
 
9
9
  class Base
10
+ # Nothing read from a page is an image, a player or a font, and the
11
+ # album pages are twenty per label; not fetching them is most of the
12
+ # time and bandwidth a scrape used to cost.
13
+ BLOCKED_RESOURCES = %w[image media font].freeze
14
+
10
15
  def initialize(url:, browser:, max_tasks: 2)
11
16
  @url = url
12
17
  @browser = browser
13
18
  @page = browser.new_page
19
+ @page.route("**/*", ->(route, request) { BLOCKED_RESOURCES.include?(request.resource_type) ? route.abort : route.continue })
14
20
  @max_tasks = max_tasks
15
21
  end
16
22
 
@@ -20,6 +26,13 @@ module BandcampDiscover
20
26
 
21
27
  private
22
28
 
29
+ # Everything read is in the server-rendered HTML, so the DOM is enough;
30
+ # waiting for "load" waited on every tracker and player asset, and one
31
+ # slow one failed the whole label.
32
+ def visit(url)
33
+ @page.goto(url, waitUntil: "domcontentloaded")
34
+ end
35
+
23
36
  def guarded
24
37
  yield
25
38
  rescue Playwright::TimeoutError => e
@@ -23,6 +23,11 @@ module BandcampDiscover
23
23
  # "a" is albums, "s" is subscriptions.
24
24
  RESULT_TYPES = %w[a s].freeze
25
25
 
26
+ # Bandcamp renamed the response fields from snake_case to camelCase in
27
+ # November 2026 while the request stayed snake_case. Both are read in case
28
+ # it rolls back or only part of their fleet has it.
29
+ BAND_URL_KEYS = %w[bandUrl band_url].freeze
30
+
26
31
  class ResponseError < StandardError; end
27
32
 
28
33
  # browser: is unused, kept so existing callers do not break.
@@ -69,7 +74,7 @@ module BandcampDiscover
69
74
  results = body["results"]
70
75
  break if results.empty?
71
76
 
72
- urls.concat(results.map { root_url(_1["band_url"]) }.compact)
77
+ urls.concat(results.map { root_url(band_url(_1)) }.compact)
73
78
 
74
79
  cursor = body["cursor"]
75
80
  break if cursor.nil? || cursor.empty?
@@ -115,12 +120,16 @@ module BandcampDiscover
115
120
  raise ResponseError, "discover_web response has no results array (keys: #{body.is_a?(Hash) ? body.keys.inspect : body.class})"
116
121
  end
117
122
 
118
- return if body["results"].empty? || body["results"].any? { _1.is_a?(Hash) && _1.key?("band_url") }
123
+ return if body["results"].empty? || body["results"].any? { _1.is_a?(Hash) && _1.keys.intersect?(BAND_URL_KEYS) }
124
+
125
+ raise ResponseError, "discover_web results carry no band url (looked for #{BAND_URL_KEYS.join(", ")}; keys: #{body["results"].first.keys.inspect})"
126
+ end
119
127
 
120
- raise ResponseError, "discover_web results carry no band_url (keys: #{body["results"].first.keys.inspect})"
128
+ def band_url(result)
129
+ result.values_at(*BAND_URL_KEYS).compact.first
121
130
  end
122
131
 
123
- # Strips the ?from=discover_page band_url carries.
132
+ # Strips the ?from=discover_page the band url carries.
124
133
  def root_url(band_url)
125
134
  return nil if band_url.nil? || band_url.empty?
126
135
 
@@ -8,7 +8,7 @@ module BandcampDiscover
8
8
  def scrape(force: false)
9
9
  super do |page|
10
10
  puts "starting to scrape #{@url}"
11
- page.goto(@url)
11
+ visit(@url)
12
12
  bio_container = page.wait_for_selector("#bio-container")
13
13
  bio_text = bio_container.query_selector("#bio-text")
14
14
 
@@ -2,6 +2,7 @@ require_relative "base"
2
2
  require_relative "album"
3
3
  require_relative "../roster"
4
4
  require "async"
5
+ require "json"
5
6
  require "async/semaphore"
6
7
 
7
8
  module BandcampDiscover
@@ -23,18 +24,33 @@ module BandcampDiscover
23
24
  # The grid alone tells a label from an artist (see Roster), and it is one
24
25
  # page load against the twenty behind albums, so callers can look at it
25
26
  # before paying for the rest.
27
+ #
28
+ # Bandcamp renders the first sixteen items as HTML and ships the rest as
29
+ # JSON in data-client-items for its own script to render after load. Read
30
+ # both and no script has to run: the grid is complete at DOMContentLoaded.
26
31
  def grid
27
32
  guarded do
28
- @page.goto(@url)
29
- items = @page.wait_for_selector("#music-grid").query_selector_all("li.music-grid-item")
30
-
31
- items.map do |item|
32
- {
33
- url: absolute(item.query_selector("a")[:href]),
34
- credit: item.query_selector(".artist-override")&.inner_text&.strip
35
- }
33
+ visit(@url)
34
+ grid = @page.wait_for_selector("#music-grid")
35
+
36
+ rendered = grid.query_selector_all("li.music-grid-item").map do |item|
37
+ {url: item.query_selector("a")[:href], credit: item.query_selector(".artist-override")&.inner_text}
36
38
  end
39
+
40
+ merge_grid(rendered, grid.get_attribute("data-client-items"))
41
+ end
42
+ end
43
+
44
+ # A credit is present in the JSON only when the release is not the
45
+ # owner's; the same convention as the rendered override.
46
+ def merge_grid(rendered, client_items_json)
47
+ client = JSON.parse(client_items_json.to_s.empty? ? "[]" : client_items_json).map do |item|
48
+ {url: item["page_url"], credit: item["artist"]}
37
49
  end
50
+
51
+ (rendered + client)
52
+ .map { |item| {url: absolute(item[:url]), credit: credit_or_nil(item[:credit])} }
53
+ .uniq { |item| item[:url] }
38
54
  end
39
55
 
40
56
  def roster(grid, band_name:)
@@ -79,6 +95,11 @@ module BandcampDiscover
79
95
  def absolute(href)
80
96
  href.start_with?("https://") ? href : "#{@base_url}#{href}"
81
97
  end
98
+
99
+ def credit_or_nil(credit)
100
+ text = credit.to_s.strip
101
+ text.empty? ? nil : text
102
+ end
82
103
  end
83
104
  end
84
105
  end
@@ -1,3 +1,3 @@
1
1
  module BandcampDiscover
2
- VERSION = "0.7.2"
2
+ VERSION = "0.8.1"
3
3
  end
@@ -14,7 +14,7 @@ class DiscoverTest < Minitest::Test
14
14
 
15
15
  def batch(band_urls, cursor: nil)
16
16
  {
17
- "results" => band_urls.map { {"band_url" => _1, "item_type" => "a"} },
17
+ "results" => band_urls.map { {"bandUrl" => _1, "itemType" => "a"} },
18
18
  "cursor" => cursor
19
19
  }
20
20
  end
@@ -71,11 +71,19 @@ class DiscoverTest < Minitest::Test
71
71
  assert_match(/no results array/, error.message)
72
72
  end
73
73
 
74
- def test_raises_when_results_no_longer_carry_band_url
75
- scraper = build(batches: [{"results" => [{"item_id" => 1, "band_name" => "x"}], "cursor" => nil}])
74
+ def test_raises_when_results_no_longer_carry_a_band_url
75
+ scraper = build(batches: [{"results" => [{"itemId" => 1, "bandName" => "x"}], "cursor" => nil}])
76
76
 
77
77
  error = assert_raises(Discover::ResponseError) { collect(scraper) }
78
- assert_match(/no band_url/, error.message)
78
+ assert_match(/no band url/, error.message)
79
+ assert_match(/itemId/, error.message)
80
+ end
81
+
82
+ # The spelling the API used until November 2026.
83
+ def test_still_reads_snake_case_results
84
+ scraper = build(batches: [{"results" => [{"band_url" => "https://a.bandcamp.com?from=discover_page"}], "cursor" => nil}])
85
+
86
+ assert_equal ["https://a.bandcamp.com"], collect(scraper)
79
87
  end
80
88
 
81
89
  def test_empty_results_is_not_an_error
data/test/music_test.rb CHANGED
@@ -6,8 +6,12 @@ require "bandcamp-discover/scrapers/music"
6
6
  class MusicTest < Minitest::Test
7
7
  Music = BandcampDiscover::Scrapers::Music
8
8
 
9
+ FakePage = Struct.new(:routes) do
10
+ def route(pattern, handler) = (self.routes ||= []) << pattern
11
+ end
12
+
9
13
  FakeBrowser = Struct.new(:pages) do
10
- def new_page = nil
14
+ def new_page = FakePage.new
11
15
  end
12
16
 
13
17
  FakeAlbum = Struct.new(:url) do
@@ -43,4 +47,39 @@ class MusicTest < Minitest::Test
43
47
 
44
48
  assert_equal [{url: nil, title: nil, tags: nil, player_url: nil, artist: "Helen"}], albums
45
49
  end
50
+
51
+ # Bandcamp renders sixteen items and ships the rest as JSON for its own
52
+ # script; reading both makes the grid complete without running it.
53
+ def test_grid_merges_rendered_items_with_the_client_items_json
54
+ rendered = [
55
+ {url: "/album/overflowing", credit: " Early Moon "},
56
+ {url: "/album/own-release", credit: nil}
57
+ ]
58
+ json = '[{"art_id":1,"artist":"David Cordero","band_id":3141249859,"id":2,"page_url":"/album/and-stillness-came","title":"And Stillness Came","type":"album"},' \
59
+ '{"art_id":3,"band_id":3141249859,"id":4,"page_url":"/track/pay","title":"Pay","type":"track"},' \
60
+ '{"art_id":5,"artist":"","band_id":3141249859,"id":6,"page_url":"/album/overflowing","title":"dup","type":"album"}]'
61
+
62
+ grid = @music.merge_grid(rendered, json)
63
+
64
+ assert_equal [
65
+ {url: "https://kranky.bandcamp.com/album/overflowing", credit: "Early Moon"},
66
+ {url: "https://kranky.bandcamp.com/album/own-release", credit: nil},
67
+ {url: "https://kranky.bandcamp.com/album/and-stillness-came", credit: "David Cordero"},
68
+ {url: "https://kranky.bandcamp.com/track/pay", credit: nil}
69
+ ], grid
70
+ end
71
+
72
+ def test_grid_without_client_items_is_the_rendered_items
73
+ rendered = [{url: "https://radiohead.bandcamp.com/album/kid-a-mnesia", credit: nil}]
74
+
75
+ assert_equal rendered, @music.merge_grid(rendered, nil)
76
+ assert_equal rendered, @music.merge_grid(rendered, "")
77
+ end
78
+
79
+ def test_pages_block_images_media_and_fonts
80
+ page = @music.instance_variable_get(:@page)
81
+
82
+ assert_equal ["**/*"], page.routes
83
+ assert_equal %w[image media font], BandcampDiscover::Scrapers::Base::BLOCKED_RESOURCES
84
+ end
46
85
  end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bandcamp-discover
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 0.8.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julian RUbisch
8
8
  bindir: bin
9
9
  cert_chain: []
10
- date: 2026-09-04 00:00:00.000000000 Z
10
+ date: 2026-11-19 00:00:00.000000000 Z
11
11
  dependencies:
12
12
  - !ruby/object:Gem::Dependency
13
13
  name: rake