bandcamp-discover 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '094dffa15caad4bbabef29bd4b3d87789425e4736c2d06461d6917efc8fe600d'
-  data.tar.gz: de1a1d20425f1ecb984878b10541208849abbf7300cd46f80834e5197730ef1c
+  metadata.gz: 4591eadf76413cf74687ca5c4233890ff1c2e96d641dd8d966c8a1336935e00d
+  data.tar.gz: cbf6cb74fca34d127a78469106af4d89597d04f1a812cd1f748ef96e0ae186b0
 SHA512:
-  metadata.gz: a6a057d4869da1039b00b9c2ffd5b4019123c4ee055cf392fd3fefc4b8fa7f427d2346081554e960a5416c95b52d507d9244912112dada5d0ddedec06820ac33
-  data.tar.gz: ae37374ebc23a193aedaed32cd741f01af048502cb9853596d6e9bf1438ec2040fcacdaee67f226951accf6bd9d674191faf8667f5bf86cc3b9216d84a408bbc
+  metadata.gz: 9eba8a9fbb1ebf9c31195bb2c4cfda3d00601d66dc4773c9590bc3e2368585e0522734b4c6f3540e27e28b13a9314a770535d893070edfcd5444ed99b8dbb49d
+  data.tar.gz: 6d5a34e8edd6c23cd1db88803e8c392911f96eb1a20eea31fe2fb31ed4258c423dc2e7ff14e243255a431e7f529eff3e20f2870bfe58deef6847a4f6c5f7c074

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    bandcamp-discover (0.7.2)
+    bandcamp-discover (0.8.0)
       async
       base64
       concurrent-ruby

data/README.rdoc

@@ -21,6 +21,10 @@ bio for the model and returned as +artists+ in the scrape result; each
 entry in +albums+ carries its own credit as +artist+ (nil when the release
 is credited to the page owner).
 
+Pages are read at DOMContentLoaded and images, media and fonts are never
+fetched: everything the scrapers use is in the server-rendered HTML, and the
+/music grid is completed from the JSON Bandcamp ships for its own script.
+
 Without a token, +label?+ falls back to matching /label|platform|records/i
 and +accepts_demos?+ is always false.
 

data/lib/bandcamp-discover/scrapers/album.rb

@@ -1,12 +1,12 @@
-require_relative "./base"
-require_relative "./music"
+require_relative "base"
+require_relative "music"
 
 module BandcampDiscover
   module Scrapers
     class Album < Base
       def scrape(force: false)
         super do |page|
-          page.goto(@url)
+          visit(@url)
 
           title = page.query_selector("meta[name=title]")&.[](:content)
 

data/lib/bandcamp-discover/scrapers/base.rb

@@ -7,10 +7,16 @@ module BandcampDiscover
     class ScrapeError < StandardError; end
 
     class Base
+      # Nothing read from a page is an image, a player or a font, and the
+      # album pages are twenty per label; not fetching them is most of the
+      # time and bandwidth a scrape used to cost.
+      BLOCKED_RESOURCES = %w[image media font].freeze
+
       def initialize(url:, browser:, max_tasks: 2)
         @url = url
         @browser = browser
         @page = browser.new_page
+        @page.route("**/*", ->(route, request) { BLOCKED_RESOURCES.include?(request.resource_type) ? route.abort : route.continue })
         @max_tasks = max_tasks
       end
 
@@ -20,6 +26,13 @@ module BandcampDiscover
 
       private
 
+      # Everything read is in the server-rendered HTML, so the DOM is enough;
+      # waiting for "load" waited on every tracker and player asset, and one
+      # slow one failed the whole label.
+      def visit(url)
+        @page.goto(url, waitUntil: "domcontentloaded")
+      end
+
       def guarded
         yield
       rescue Playwright::TimeoutError => e

data/lib/bandcamp-discover/scrapers/label.rb

@@ -8,7 +8,7 @@ module BandcampDiscover
       def scrape(force: false)
         super do |page|
           puts "starting to scrape #{@url}"
-          page.goto(@url)
+          visit(@url)
           bio_container = page.wait_for_selector("#bio-container")
           bio_text = bio_container.query_selector("#bio-text")
 

data/lib/bandcamp-discover/scrapers/music.rb

@@ -2,6 +2,7 @@ require_relative "base"
 require_relative "album"
 require_relative "../roster"
 require "async"
+require "json"
 require "async/semaphore"
 
 module BandcampDiscover
@@ -23,18 +24,33 @@ module BandcampDiscover
       # The grid alone tells a label from an artist (see Roster), and it is one
       # page load against the twenty behind albums, so callers can look at it
       # before paying for the rest.
+      #
+      # Bandcamp renders the first sixteen items as HTML and ships the rest as
+      # JSON in data-client-items for its own script to render after load. Read
+      # both and no script has to run: the grid is complete at DOMContentLoaded.
       def grid
         guarded do
-          @page.goto(@url)
-          items = @page.wait_for_selector("#music-grid").query_selector_all("li.music-grid-item")
-
-          items.map do |item|
-            {
-              url: absolute(item.query_selector("a")[:href]),
-              credit: item.query_selector(".artist-override")&.inner_text&.strip
-            }
+          visit(@url)
+          grid = @page.wait_for_selector("#music-grid")
+
+          rendered = grid.query_selector_all("li.music-grid-item").map do |item|
+            {url: item.query_selector("a")[:href], credit: item.query_selector(".artist-override")&.inner_text}
           end
+
+          merge_grid(rendered, grid.get_attribute("data-client-items"))
+        end
+      end
+
+      # A credit is present in the JSON only when the release is not the
+      # owner's; the same convention as the rendered override.
+      def merge_grid(rendered, client_items_json)
+        client = JSON.parse(client_items_json.to_s.empty? ? "[]" : client_items_json).map do |item|
+          {url: item["page_url"], credit: item["artist"]}
         end
+
+        (rendered + client)
+          .map { |item| {url: absolute(item[:url]), credit: credit_or_nil(item[:credit])} }
+          .uniq { |item| item[:url] }
       end
 
       def roster(grid, band_name:)
@@ -79,6 +95,11 @@ module BandcampDiscover
       def absolute(href)
         href.start_with?("https://") ? href : "#{@base_url}#{href}"
       end
+
+      def credit_or_nil(credit)
+        text = credit.to_s.strip
+        text.empty? ? nil : text
+      end
     end
   end
 end

data/lib/bandcamp-discover/version.rb

@@ -1,3 +1,3 @@
 module BandcampDiscover
-  VERSION = "0.7.2"
+  VERSION = "0.8.0"
 end

data/test/music_test.rb

@@ -6,8 +6,12 @@ require "bandcamp-discover/scrapers/music"
 class MusicTest < Minitest::Test
   Music = BandcampDiscover::Scrapers::Music
 
+  FakePage = Struct.new(:routes) do
+    def route(pattern, handler) = (self.routes ||= []) << pattern
+  end
+
   FakeBrowser = Struct.new(:pages) do
-    def new_page = nil
+    def new_page = FakePage.new
   end
 
   FakeAlbum = Struct.new(:url) do
@@ -43,4 +47,39 @@ class MusicTest < Minitest::Test
 
     assert_equal [{url: nil, title: nil, tags: nil, player_url: nil, artist: "Helen"}], albums
   end
+
+  # Bandcamp renders sixteen items and ships the rest as JSON for its own
+  # script; reading both makes the grid complete without running it.
+  def test_grid_merges_rendered_items_with_the_client_items_json
+    rendered = [
+      {url: "/album/overflowing", credit: " Early Moon "},
+      {url: "/album/own-release", credit: nil}
+    ]
+    json = '[{"art_id":1,"artist":"David Cordero","band_id":3141249859,"id":2,"page_url":"/album/and-stillness-came","title":"And Stillness Came","type":"album"},' \
+           '{"art_id":3,"band_id":3141249859,"id":4,"page_url":"/track/pay","title":"Pay","type":"track"},' \
+           '{"art_id":5,"artist":"","band_id":3141249859,"id":6,"page_url":"/album/overflowing","title":"dup","type":"album"}]'
+
+    grid = @music.merge_grid(rendered, json)
+
+    assert_equal [
+      {url: "https://kranky.bandcamp.com/album/overflowing", credit: "Early Moon"},
+      {url: "https://kranky.bandcamp.com/album/own-release", credit: nil},
+      {url: "https://kranky.bandcamp.com/album/and-stillness-came", credit: "David Cordero"},
+      {url: "https://kranky.bandcamp.com/track/pay", credit: nil}
+    ], grid
+  end
+
+  def test_grid_without_client_items_is_the_rendered_items
+    rendered = [{url: "https://radiohead.bandcamp.com/album/kid-a-mnesia", credit: nil}]
+
+    assert_equal rendered, @music.merge_grid(rendered, nil)
+    assert_equal rendered, @music.merge_grid(rendered, "")
+  end
+
+  def test_pages_block_images_media_and_fonts
+    page = @music.instance_variable_get(:@page)
+
+    assert_equal ["**/*"], page.routes
+    assert_equal %w[image media font], BandcampDiscover::Scrapers::Base::BLOCKED_RESOURCES
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: bandcamp-discover
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - Julian RUbisch
 bindir: bin
 cert_chain: []
-date: 2026-09-04 00:00:00.000000000 Z
+date: 2026-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake