bandcamp-discover 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a1572032feb21d35545c35c0c35d800e277816d1506370dfe45c67b0c155b98
-  data.tar.gz: 5d33aa525fce53607adaf8f5aef417e24203b165fb93e1b2ffd6af3ad7dc64ce
+  metadata.gz: c2c7501fed34fb2971b6be16d1abb01ea93b25246dc117d0ef5f45f76a884a81
+  data.tar.gz: f3b13208003d5c7473a5fab3c010119f0cafb6cd5e161f1c7a92796af97ed8fa
 SHA512:
-  metadata.gz: 4d52699fd00743a73a4508a239169b11070a1aac78f8c7f49b0a614a689d06b29fdae2495c6f38af6a85c69a0659953eb3c96091f95f398331024763288d6e06
-  data.tar.gz: 25b6bcc92e173955671f6e441357423e51e42ad0cd1219029b7faed8ae6e395d8b08c8932fc1851361f62357ca1bf372c84b9459e77934a68e4dd8d35538754c
+  metadata.gz: 39337db593351f00eb19f50826342512bd4e7e16fb680074d30734669486db1f56d9e6b7959664acf1ffe77d122984571eabf4c261a02abf52e778c9dbab0fcb
+  data.tar.gz: 32c9359e36d955b34fb9fc36942bbe455dc99e4a41e9032ac0ae3d6420e76bc9b8840f629c90e797669358979b2317bcd42ef55bf54e52ab07f00378ed637f45

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    bandcamp-discover (0.1.7)
+    bandcamp-discover (0.5.0)
       async
       base64
       concurrent-ruby

data/lib/bandcamp-discover/scrapers/base.rb

@@ -1,7 +1,11 @@
-require 'playwright'
+require "playwright"
 
 module BandcampDiscover
   module Scrapers
+    # Previously rescued into a puts + nil return, which made a total scraping
+    # outage read as "no results" and go unnoticed.
+    class ScrapeError < StandardError; end
+
     class Base
       def initialize(url:, browser:, max_tasks: 2)
         @url = url
@@ -12,8 +16,8 @@ module BandcampDiscover
 
       def scrape(force: false)
         yield @page if block_given?
-      rescue Playwright::TimeoutError
-        puts "Failed to wait for element"
+      rescue Playwright::TimeoutError => e
+        raise ScrapeError, "Timed out waiting for an element on #{@url}: #{e.message}"
       end
     end
   end

data/lib/bandcamp-discover/scrapers/discover.rb

@@ -1,5 +1,6 @@
-require_relative "base"
 require_relative "label"
+require "net/http"
+require "json"
 require "uri"
 require "async"
 require "async/semaphore"
@@ -8,43 +9,127 @@ require "concurrent"
 
 module BandcampDiscover
   module Scrapers
-    class Discover < Base
-      def initialize(genre:, browser:, max_tasks: Concurrent.processor_count)
-        super(url: "https://bandcamp.com/discover/#{genre}?s=rand", browser: browser, max_tasks: max_tasks)
+    # Not Playwright: the /discover/<genre> grid only exists after the Vue app
+    # hydrates, and Fastly serves that page a JS challenge to datacenter IPs that
+    # headless Chrome never clears. The XHR behind it is unchallenged.
+    #
+    # It is also private and unversioned, hence assert_shape!.
+    class Discover
+      API_URI = URI("https://bandcamp.com/api/discover/1/discover_web")
+      PAGE_SIZE = 60
+      OPEN_TIMEOUT = 10
+      READ_TIMEOUT = 30
+
+      # "a" is albums, "s" is subscriptions.
+      RESULT_TYPES = %w[a s].freeze
+
+      class ResponseError < StandardError; end
+
+      # browser: is unused, kept so existing callers do not break.
+      def initialize(genre:, browser: nil, max_tasks: Concurrent.processor_count, pages: 1)
+        @genre = genre
+        @browser = browser
+        @max_tasks = max_tasks
+        @pages = pages
       end
 
       def scrape(force: false)
-        super do |page|
-          page.goto(@url)
+        urls = band_urls
 
-          records_list = page.wait_for_selector("ul.items")
-          records = records_list.query_selector_all("li")
-          links = records.map { _1.query_selector("a")[:href] }
+        barrier = Async::Barrier.new
 
-          # click "View more results"
+        Sync do
+          semaphore = Async::Semaphore.new(@max_tasks, parent: barrier)
 
-          uris = links.map { ::URI.parse(_1) }
+          urls.map do |url|
+            semaphore.async do
+              if block_given?
+                yield url
+              else
+                Scrapers::Label.new(url: url, browser: @browser).scrape
+              end
+            end
+          end.map(&:wait).compact
+        ensure
+          barrier.stop
+        end
+      end
 
-          barrier = Async::Barrier.new
+      private
 
-          Sync do
-            semaphore = Async::Semaphore.new(@max_tasks, parent: barrier)
+      # One label recurs across its albums, so dedupe.
+      def band_urls
+        cursor = "*"
+        urls = []
 
-            uris.map do |uri|
-              url = "#{uri.scheme}://#{uri.host}"
+        @pages.times do
+          body = fetch_batch(cursor)
+          assert_shape!(body)
 
-              semaphore.async do
-                if block_given?
-                  yield url
-                else
-                  Scrapers::Label.new(url: url, browser: @browser).scrape
-                end
-              end
-            end.map(&:wait).compact
-          ensure
-            barrier.stop
-          end
+          results = body["results"]
+          break if results.empty?
+
+          urls.concat(results.map { root_url(_1["band_url"]) }.compact)
+
+          cursor = body["cursor"]
+          break if cursor.nil? || cursor.empty?
         end
+
+        urls.uniq
+      end
+
+      def fetch_batch(cursor)
+        request = Net::HTTP::Post.new(API_URI)
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate(
+          category_id: 0,
+          tag_norm_names: [@genre],
+          geoname_id: 0,
+          slice: "rand",
+          time_facet_id: nil,
+          cursor: cursor,
+          size: PAGE_SIZE,
+          include_result_types: RESULT_TYPES,
+          followed_bands: false
+        )
+
+        response = Net::HTTP.start(
+          API_URI.hostname,
+          API_URI.port,
+          use_ssl: true,
+          open_timeout: OPEN_TIMEOUT,
+          read_timeout: READ_TIMEOUT
+        ) { _1.request(request) }
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise ResponseError, "discover_web returned #{response.code} for genre #{@genre.inspect}"
+        end
+
+        JSON.parse(response.body)
+      rescue JSON::ParserError => e
+        raise ResponseError, "discover_web returned unparseable JSON for genre #{@genre.inspect}: #{e.message}"
+      end
+
+      def assert_shape!(body)
+        unless body.is_a?(Hash) && body["results"].is_a?(Array)
+          raise ResponseError, "discover_web response has no results array (keys: #{body.is_a?(Hash) ? body.keys.inspect : body.class})"
+        end
+
+        return if body["results"].empty? || body["results"].any? { _1.is_a?(Hash) && _1.key?("band_url") }
+
+        raise ResponseError, "discover_web results carry no band_url (keys: #{body["results"].first.keys.inspect})"
+      end
+
+      # Strips the ?from=discover_page band_url carries.
+      def root_url(band_url)
+        return nil if band_url.nil? || band_url.empty?
+
+        uri = URI.parse(band_url)
+        return nil if uri.scheme.nil? || uri.host.nil?
+
+        "#{uri.scheme}://#{uri.host}"
+      rescue URI::InvalidURIError
+        nil
       end
     end
   end

data/lib/bandcamp-discover/version.rb

@@ -1,3 +1,3 @@
 module BandcampDiscover
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/test/discover_test.rb

@@ -0,0 +1,86 @@
+require_relative "test_helper"
+require "bandcamp-discover/scrapers/discover"
+
+class DiscoverTest < Minitest::Test
+  Discover = BandcampDiscover::Scrapers::Discover
+
+  # Stub only the network, so the rest of the class runs for real.
+  def build(genre: "ambient", pages: 1, batches: [])
+    scraper = Discover.new(genre: genre, max_tasks: 2, pages: pages)
+    queue = batches.dup
+    scraper.define_singleton_method(:fetch_batch) { |_cursor| queue.shift }
+    scraper
+  end
+
+  def batch(band_urls, cursor: nil)
+    {
+      "results" => band_urls.map { {"band_url" => _1, "item_type" => "a"} },
+      "cursor" => cursor
+    }
+  end
+
+  def collect(scraper)
+    [].tap { |out| scraper.scrape { |url| out << url } }
+  end
+
+  def test_yields_scheme_and_host_only
+    scraper = build(batches: [batch(["https://polarseasrecordings.bandcamp.com?from=discover_page"])])
+
+    assert_equal ["https://polarseasrecordings.bandcamp.com"], collect(scraper)
+  end
+
+  def test_dedupes_labels_appearing_under_several_albums
+    scraper = build(batches: [batch([
+      "https://a.bandcamp.com?from=discover_page",
+      "https://a.bandcamp.com?from=discover_page",
+      "https://b.bandcamp.com?from=discover_page"
+    ])])
+
+    assert_equal ["https://a.bandcamp.com", "https://b.bandcamp.com"], collect(scraper).sort
+  end
+
+  def test_follows_cursor_across_pages
+    scraper = build(pages: 2, batches: [
+      batch(["https://a.bandcamp.com"], cursor: "next"),
+      batch(["https://b.bandcamp.com"], cursor: nil)
+    ])
+
+    assert_equal ["https://a.bandcamp.com", "https://b.bandcamp.com"], collect(scraper).sort
+  end
+
+  def test_stops_early_when_cursor_is_exhausted
+    scraper = build(pages: 3, batches: [
+      batch(["https://a.bandcamp.com"], cursor: nil),
+      batch(["https://never-reached.bandcamp.com"])
+    ])
+
+    assert_equal ["https://a.bandcamp.com"], collect(scraper)
+  end
+
+  def test_skips_unusable_band_urls
+    scraper = build(batches: [batch([nil, "", "not a url", "/relative", "https://ok.bandcamp.com"])])
+
+    assert_equal ["https://ok.bandcamp.com"], collect(scraper)
+  end
+
+  # A silent zero-result discovery is how the last breakage hid; these must raise.
+  def test_raises_when_results_array_is_missing
+    scraper = build(batches: [{"cursor" => "*"}])
+
+    error = assert_raises(Discover::ResponseError) { collect(scraper) }
+    assert_match(/no results array/, error.message)
+  end
+
+  def test_raises_when_results_no_longer_carry_band_url
+    scraper = build(batches: [{"results" => [{"item_id" => 1, "band_name" => "x"}], "cursor" => nil}])
+
+    error = assert_raises(Discover::ResponseError) { collect(scraper) }
+    assert_match(/no band_url/, error.message)
+  end
+
+  def test_empty_results_is_not_an_error
+    scraper = build(batches: [{"results" => [], "cursor" => nil}])
+
+    assert_empty collect(scraper)
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: bandcamp-discover
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Julian RUbisch
 bindir: bin
 cert_chain: []
-date: 2026-02-07 00:00:00.000000000 Z
+date: 2026-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -202,6 +202,7 @@ files:
 - lib/bandcamp-discover/scrapers/music.rb
 - lib/bandcamp-discover/version.rb
 - test/default_test.rb
+- test/discover_test.rb
 - test/test_helper.rb
 homepage: https://julianrubisch.at
 licenses: []