bandcamp-discover 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 82fa227242a4bf5508d6391e6791e2a6f45b041eb9b05c824b94fe4e5aa15cd0
+  data.tar.gz: d2e8480b5267e584a05d2af1c36d16ad7ea6d9af5416fe094c73c7405723a778
+SHA512:
+  metadata.gz: 7e5ecead1e89a8a9995df2dc860bfe2db7e5935b876f7fb4119ec6821adecd9b9ba8a18834bfdbdb877cc2cd82449c1dad44b36d724b794733edd32271fbeb3a
+  data.tar.gz: bd904b22d2ae10650a8bed975df5e6c3caa7d1a64624dcb2d5a1da1dc568a52f77ba3717a45162e9d80845ef2f88315fe8e4a1daf72caf28a9c91261052dccbb

data/.gitignore

@@ -0,0 +1,3 @@
+*~
+*#*#
+/pkg/

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.rdoc

@@ -0,0 +1,6 @@
+= bandcamp-discover
+
+Describe your project here
+
+:include:bandcamp-discover.rdoc
+

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rake/clean'
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+
+require "bundler/gem_tasks"
+
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+  rd.title = 'Your application title'
+end
+
+spec = Gem::Specification.load("bandcamp-discover.gemspec")
+
+Gem::PackageTask.new(spec) do |pkg|
+end
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+end
+
+task :default => :test

data/bandcamp-discover.gemspec

@@ -0,0 +1,31 @@
+# Ensure we require the local version and not one we might have installed already
+require File.join([File.dirname(__FILE__),'lib','bandcamp-discover','version.rb'])
+spec = Gem::Specification.new do |s|
+  s.name = 'bandcamp-discover'
+  s.version = BandcampDiscover::VERSION
+  s.author = 'Julian RUbisch'
+  s.email = 'julian@julianrubisch.at'
+  s.homepage = 'https://julianrubisch.at'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Uses Playwright to scrape bandcamp labels'
+  s.files = `git ls-files`.split("
+")
+  s.require_paths << 'lib'
+  s.extra_rdoc_files = ['README.rdoc','bandcamp-discover.rdoc']
+  s.rdoc_options << '--title' << 'bandcamp-discover' << '--main' << 'README.rdoc' << '-ri'
+  s.bindir = 'bin'
+  s.executables << 'bandcamp-discover'
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rdoc')
+  s.add_development_dependency('minitest')
+  s.add_development_dependency('standard')
+  s.add_runtime_dependency('gli','~> 2.21.5')
+
+  s.add_dependency('playwright-ruby-client')
+  s.add_dependency('sqlite3')
+  s.add_dependency('async')
+  s.add_dependency('concurrent-ruby')
+  s.add_dependency('base64')
+  s.add_dependency('logger')
+  s.add_dependency('ostruct')
+end

data/bandcamp-discover.rdoc

@@ -0,0 +1,5 @@
+= bandcamp-discover
+
+Generate this with
+    bandcamp-discover _doc
+After you have described your command line interface

data/bin/bandcamp-discover

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+require 'gli'
+require 'bandcamp-discover'
+
+class App
+  extend GLI::App
+
+  program_desc 'Describe your application here'
+
+  version BandcampDiscover::VERSION
+
+  subcommand_option_handling :normal
+  arguments :strict
+
+  desc 'Scrape bandcamp discover, labels, or albums'
+  # arg_name 'Describe arguments to scrape here'
+  command :scrape do |c|
+    c.desc 'Use a headless browser'
+    c.switch :headless, default_value: true
+
+    c.desc 'Maximum tasks for scraping a label'
+    c.flag :"max-tasks", default_value: 2
+
+    c.arg "genre"
+    c.command :discover do |discover|
+      discover.action do |global_options,options,args|
+        genre = args.shift
+        genre ||= "experimental"
+
+        Playwright.create(playwright_cli_executable_path: 'npx playwright') do |playwright|
+          playwright.chromium.launch(headless: options["headless"]) do |browser|
+            puts BandcampDiscover::Scrapers::Discover.new(genre:, browser:).scrape
+          end
+        end
+
+        # If you have any errors, just raise them
+        # raise "that command made no sense"
+      end
+    end
+
+    c.arg "url", required: true
+    c.command :label do |label|
+      label.action do |global_options,options,args|
+        url = args.shift
+
+        Playwright.create(playwright_cli_executable_path: 'npx playwright') do |playwright|
+          playwright.chromium.launch(headless: options["headless"]) do |browser|
+            puts BandcampDiscover::Scrapers::Label.new(url:, browser:, max_tasks: options.values.first["max-tasks"].to_i).scrape
+          end
+        end
+
+        # If you have any errors, just raise them
+        # raise "that command made no sense"
+      end
+    end
+  end
+
+  pre do |global,command,options,args|
+    # Pre logic here
+    # Return true to proceed; false to abort and not call the
+    # chosen command
+    # Use skips_pre before a command to skip this block
+    # on that command only
+    true
+  end
+
+  post do |global,command,options,args|
+    # Post logic here
+    # Use skips_post before a command to skip this
+    # block on that command only
+  end
+
+  on_error do |exception|
+    # Error logic here
+    # return false to skip default error handling
+    true
+  end
+end
+
+exit App.run(ARGV)

data/lib/bandcamp-discover/scrapers/album.rb

@@ -0,0 +1,18 @@
+require_relative "./base"
+require_relative "./music"
+
+module BandcampDiscover
+  module Scrapers
+    class Album < Base
+      def scrape
+        super do |page|
+          page.goto(@url)
+
+          # querying all tags in the bottom and returning their text node
+          tags = page.query_selector_all("a.tag")
+          tags.map { _1.inner_text }
+        end
+      end
+    end
+  end
+end

data/lib/bandcamp-discover/scrapers/base.rb

@@ -0,0 +1,20 @@
+require 'playwright'
+
+module BandcampDiscover
+  module Scrapers
+    class Base
+      def initialize(url:, browser:, max_tasks: 2)
+        @url = url
+        @browser = browser
+        @page = browser.new_page
+        @max_tasks = max_tasks
+      end
+
+      def scrape
+        yield @page if block_given?
+      rescue Playwright::TimeoutError
+        puts "Failed to wait for element"
+      end
+    end
+  end
+end

data/lib/bandcamp-discover/scrapers/discover.rb

@@ -0,0 +1,47 @@
+require_relative "base"
+require_relative "label"
+require "uri"
+require "async"
+require "async/semaphore"
+require "async/barrier"
+require "concurrent"
+
+module BandcampDiscover
+  module Scrapers
+    class Discover < Base
+      def initialize(genre:, browser:, max_tasks: Concurrent.processor_count)
+        super(url: "https://bandcamp.com/discover/#{genre}?s=rand", browser: browser, max_tasks: max_tasks)
+      end
+
+      def scrape
+        super do |page|
+          page.goto(@url)
+
+          records_list = page.wait_for_selector("ul.items")
+          records = records_list.query_selector_all("li")
+          links = records.map { _1.query_selector("a")[:href] }
+
+          # click "View more results"
+
+          uris = links.map { ::URI.parse(_1) }
+
+          barrier = Async::Barrier.new
+
+          Sync do
+            semaphore = Async::Semaphore.new(@max_tasks, parent: barrier)
+
+            uris.map do |uri|
+              url = "#{uri.scheme}://#{uri.host}"
+
+              semaphore.async do
+                Scrapers::Label.new(url: url, browser: @browser).scrape
+              end
+            end.map(&:wait).compact
+          ensure
+            barrier.stop
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bandcamp-discover/scrapers/label.rb

@@ -0,0 +1,28 @@
+require_relative "./base"
+require_relative "./music"
+
+module BandcampDiscover
+  module Scrapers
+    class Label < Base
+      def scrape
+        super do |page|
+          puts "starting to scrape #{@url}"
+          page.goto(@url)
+          bio_container = page.wait_for_selector("#bio-container")
+          bio_text = bio_container.query_selector("#bio-text")
+          if bio_text&.inner_html =~ /label|platform/
+            return Sync do
+              music_tags = Scrapers::Music.new(url: "#{@url}/music", browser: @browser, max_tasks: @max_tasks).scrape
+
+              puts "done scraping #{@url}"
+
+              [@url, music_tags]
+            end
+          else
+            puts "not a label: #{@url}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bandcamp-discover/scrapers/music.rb

@@ -0,0 +1,48 @@
+require_relative "base"
+require_relative "album"
+require "async"
+require "async/semaphore"
+
+module BandcampDiscover
+  module Scrapers
+    class Music < Base
+      def initialize(url:, browser:, max_tasks:)
+        super
+
+        uri = URI.parse(url)
+        @base_url = "#{uri.scheme}://#{uri.host}"
+      end
+
+      def scrape
+        super do |page|
+          page.goto(@url)
+          album_list = page.wait_for_selector("#music-grid")
+          album_links = album_list.query_selector_all("li.music-grid-item > a")
+
+          semaphore = Async::Semaphore.new(@max_tasks)
+
+          album_tags = album_links.take(20).map do |album_link|
+            semaphore.async do
+              url = "#{@base_url}#{album_link[:href]}"
+              puts "starting to scrape #{url}"
+
+              tags = Scrapers::Album.new(url: url, browser: @browser).scrape
+
+              puts "done scraping #{url}"
+
+              tags
+            end
+          end.map(&:wait)
+
+          normalize_tally(album_tags.flatten.tally)
+        end
+      end
+
+      def normalize_tally(tally)
+        total = tally.values.sum.to_f
+        tally.transform_values! { |count| count / total }
+        tally.sort_by { |k, v| v }.reverse.to_h
+      end
+    end
+  end
+end

data/lib/bandcamp-discover/version.rb

@@ -0,0 +1,3 @@
+module BandcampDiscover
+  VERSION = '0.1.0'
+end

data/lib/bandcamp-discover.rb

@@ -0,0 +1,7 @@
+require 'async'
+require 'bandcamp-discover/version.rb'
+require 'bandcamp-discover/scrapers/discover.rb'
+
+# Add requires for other files you add to your project here, so
+# you just need to require this one file in your bin file
+

data/test/default_test.rb

@@ -0,0 +1,14 @@
+require_relative "test_helper"
+
+class DefaultTest < Minitest::Test
+
+  def setup
+  end
+
+  def teardown
+  end
+
+  def test_the_truth
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+
+# Add test libraries you want to use here, e.g. mocha
+# Add helper classes or methods here, too

metadata

@@ -0,0 +1,230 @@
+--- !ruby/object:Gem::Specification
+name: bandcamp-discover
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Julian RUbisch
+bindir: bin
+cert_chain: []
+date: 2025-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.21.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.21.5
+- !ruby/object:Gem::Dependency
+  name: playwright-ruby-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+email: julian@julianrubisch.at
+executables:
+- bandcamp-discover
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+- bandcamp-discover.rdoc
+files:
+- ".gitignore"
+- Gemfile
+- README.rdoc
+- Rakefile
+- bandcamp-discover.gemspec
+- bandcamp-discover.rdoc
+- bin/bandcamp-discover
+- lib/bandcamp-discover.rb
+- lib/bandcamp-discover/scrapers/album.rb
+- lib/bandcamp-discover/scrapers/base.rb
+- lib/bandcamp-discover/scrapers/discover.rb
+- lib/bandcamp-discover/scrapers/label.rb
+- lib/bandcamp-discover/scrapers/music.rb
+- lib/bandcamp-discover/version.rb
+- test/default_test.rb
+- test/test_helper.rb
+homepage: https://julianrubisch.at
+licenses: []
+metadata: {}
+rdoc_options:
+- "--title"
+- bandcamp-discover
+- "--main"
+- README.rdoc
+- "-ri"
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Uses Playwright to scrape bandcamp labels
+test_files: []