pandata 0.3.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9819160b7e403e619b6a5dfeae8e8c9199a74a4
-  data.tar.gz: b37522f882a3d40b7f0546bcea59f14aa52ec0d4
+  metadata.gz: 94c921582f5d5d1d8c56d82894b49a2903383082
+  data.tar.gz: bfca95f7246ed9a403a4127ccdb5cd40af1214c5
 SHA512:
-  metadata.gz: 9f1e76bec614fc7b251a1b85ad8cd3720b6ac1ac02bfe037f6fa65f70f48651f530c705b0d697fea815d7aca77b638cbac99ef6a6c9a5d8d649600cc4e2fb43e
-  data.tar.gz: 15b8d23fab4a7f527da0f904a2dec68c4e760c018442f86f7b3f9c547aaa88798e8797421d77c1b583f2c3b4b8adcd73b2a551f17dad19aac0e6cc4224c1c746
+  metadata.gz: 1700ebfca8412057890d4e4aa6ce68024260aad32412bf005e8b1313d19d768864db7b8726b1b0ee28d2ad899eb20d0c17e2f377a41c7cd620b2359b8c388c4e
+  data.tar.gz: 1ec60d0df1026059df3b97f04114c038c34be1647e81bf2e83fcfdae24fa09fc76e13533dafcf87030fe0cbd624d7ed225facb273262b6b9df6969241abfdef1

data/README.md

@@ -1,19 +1,16 @@
-# Pandata
+# Pandata [![Build Status](https://travis-ci.org/ustasb/pandata.svg?branch=master)](https://travis-ci.org/ustasb/pandata)
 
 Pandata is a Ruby 1.9+ library for downloading a user's Pandora.com data. This data includes:
 
-- Playing Station *
-- Recent Activity *
-- Stations *
-- Bookmarks (artists, tracks) *
 - Likes (albums, artists, stations, tracks)
 - Followers
 - Following
 
-Where possible, Pandora [feeds][1] are used (indicated by an * above).
-
 **Pandata can only access public Pandora profiles.** This option can be changed in Pandora's settings.
 
+**Note:** Scraping is a fragile task and Pandora can (and has) easily break this
+gem. Version 2 of this gem represents the removal of Pandora's [feeds][1] feature.
+
 ## Installing
 
 Pandata is a Ruby gem. To install, execute:
@@ -38,7 +35,7 @@ pandora.com/profile/\<my_webname\>
 First, create a new Pandata scraper for a user:
 
     require 'pandata'
-    
+
     # Scraper.get takes either an email or a webname.
     # Returns an array of similar webnames if no match is found.
     johns_scraper = Pandata::Scraper.get('john@example.com')
@@ -48,12 +45,6 @@ Next, start scraping!
     # Get only liked tracks
     likes = johns_scraper.likes(:tracks)
 
-    # Get all bookmarks (artists and tracks)
-    bookmarks = johns_scraper.bookmarks
-
-    # Get all stations
-    stations = johns_scraper.stations
-
     # Get all followers
     followers = johns_scraper.followers
 
@@ -75,11 +66,38 @@ For an up-to-date list, check out:
 
     pandata john@example.com --liked_tracks
 
-    # Get liked tracks, artists and bookmarked tracks + output as JSON.
-    pandata my_webname -lLb --json
+    # Get liked tracks, artists + output as JSON.
+    pandata my_webname -lL --json
 
     # Get all data and output to a file.
     pandata my_webname --all -o my_pandora_data.txt
 
-[1]: http://www.pandora.com/feeds
+### FAQ
+
+#### Q: Pandata is not grabbing all my liked tracks on Pandora. What's up with that?!
+
+First, for those coming from [pandify.com](http://pandify.com), Pandata is the
+tool that actually grabs your Pandora data.
+
+So, Pandora doesn't make it easy to retrieve users' data. This gem scrapes
+public Pandora profiles by going through a few fake proxy accounts. These fake
+accounts are shared between all Pandata users and it seems that Pandora now
+prevents those accounts from seeing some data on the website:
+
+![Unable to display thumb data.](https://raw.githubusercontent.com/ustasb/pandata/master/unable_to_display_data.png)
+
+As a workaround, I tried using the same fake accounts via the mobile endpoints.
+Pandora hasn't flagged the fake proxy accounts yet via this method. However, I've
+noticed that if you try to scroll through some user's liked tracks on Pandora's
+mobile app, the app will get stuck randomly and fail to load the next tracks.
+The loading spinner will never stop:
+
+![tconrad infinite feed](https://raw.githubusercontent.com/ustasb/pandata/master/tconrad_infinite_feed.png)
+
+*The above is Tom Conrad's liked tracks mobile feed. He has 1200+ but the feed stops at around 185.*
+
+Again, this only happens for some users and I can't do anything about it. If it
+affects you, I'm sorry :(
+
+[1]: http://blog.pandora.com/2006/02/02/pandora_21_rss
 [2]: http://rubydoc.info/gems/pandata/frames

data/lib/pandata.rb

@@ -4,17 +4,8 @@ require_relative 'pandata/data_urls'
 require_relative 'pandata/downloader'
 require_relative 'pandata/parser'
 require_relative 'pandata/scraper'
+require_relative 'pandata/version'
 
 module Pandata
   class PandataError < StandardError; end
-
-  module Version
-    MAJOR = 0
-    MINOR = 3
-    PATCH = 4
-    BUILD = nil
-
-    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
-  end
 end
-

data/lib/pandata/argv_parser.rb

@@ -23,14 +23,14 @@ module Pandata
       get_all_data = false
 
       options[:opts] = OptionParser.new do |opts|
-        opts.banner = 'Pandata: A tool for downloading Pandora.com data (likes, bookmarks, stations, etc.)'
+        opts.banner = 'Pandata: A tool for downloading Pandora.com data'
         opts.define_head 'Usage: pandata <email|webname> [options]'
         opts.separator <<-END
 
 Examples:
   pandata john@example.com --liked_tracks
   pandata my_webname --all -o my_pandora_data.txt
-  pandata my_webname -lLb --json
+  pandata my_webname -lL --json
 
 Options:
         END
@@ -39,18 +39,6 @@ Options:
           get_all_data = true
         end
 
-        opts.on('-a', '--recent_activity', 'Get recent activity') do
-          options[:data_to_get] << :recent_activity
-        end
-
-        opts.on('-B', '--bookmarked_artists', 'Get all bookmarked artists') do
-          options[:data_to_get] << :bookmarked_artists
-        end
-
-        opts.on('-b', '--bookmarked_tracks', 'Get all bookmarked tracks') do
-          options[:data_to_get] << :bookmarked_tracks
-        end
-
         opts.on('-F', '--followers', "Get all user's followers") do
           options[:data_to_get] << :followers
         end
@@ -83,14 +71,6 @@ Options:
           options[:output_file] = path
         end
 
-        opts.on('-S', '--playing_station', 'Get currently playing station') do
-          options[:data_to_get] << :playing_station
-        end
-
-        opts.on('-s', '--stations', 'Get all stations') do
-          options[:data_to_get] << :stations
-        end
-
         opts.on_tail("-h", "--help", "Show this message") do
           options[:help] = true
         end
@@ -107,11 +87,6 @@ Options:
 
       if get_all_data
         options[:data_to_get] = [
-          :recent_activity,
-          :playing_station,
-          :stations,
-          :bookmarked_tracks,
-          :bookmarked_artists,
           :liked_tracks,
           :liked_artists,
           :liked_albums,

data/lib/pandata/cli.rb

@@ -85,11 +85,9 @@ module Pandata
                      "  ** No Data **\n"
                    else
                      case category
-                     when /playing_station|recent_activity/
-                       formatter.list(cat_data)
-                     when /liked_tracks|bookmarked_tracks/
+                     when /liked_tracks/
                        formatter.tracks(cat_data)
-                     when /liked_artists|bookmarked_artists|stations|liked_stations/
+                     when /liked_artists|liked_stations/
                        formatter.sort_list(cat_data)
                      when :liked_albums
                        formatter.albums(cat_data)
@@ -109,10 +107,9 @@ module Pandata
       scraper_data = {}
 
       @data_to_get.each do |data_category|
-        if /(bookmark|like)e?d_(.*)/ =~ data_category
-          method = $1 << 's'  # 'likes' or 'bookmarks'
-          argument = $2.to_sym  # :tracks, :artists, :stations or :albums
-          scraper_data[data_category] = @scraper.public_send(method, argument)
+        if /liked_(.*)/ =~ data_category
+          argument = $1.to_sym  # :tracks, :artists, :stations or :albums
+          scraper_data[data_category] = @scraper.public_send(:likes, argument)
         else
           scraper_data[data_category] = @scraper.public_send(data_category)
         end

data/lib/pandata/data_urls.rb

@@ -5,11 +5,6 @@ module Pandata
   # URLs to Pandora's data!
   DATA_FEED_URLS = {
     user_search:          'http://www.pandora.com/content/connect?searchString=%{searchString}',
-    recent_activity:      'http://feeds.pandora.com/feeds/people/%{webname}/recentactivity.xml',
-    playing_station:      'http://feeds.pandora.com/feeds/people/%{webname}/nowplaying.xml',
-    stations:             "http://feeds.pandora.com/feeds/people/%{webname}/stations.xml?max=#{MAX_RESULTS}",
-    bookmarked_tracks:    "http://feeds.pandora.com/feeds/people/%{webname}/favorites.xml?max=#{MAX_RESULTS}",
-    bookmarked_artists:   "http://feeds.pandora.com/feeds/people/%{webname}/favoriteartists.xml?max=#{MAX_RESULTS}",
     liked_tracks:         'http://www.pandora.com/content/mobile/profile_likes_track.vm?likeStartIndex=%{nextLikeStartIndex}&thumbStartIndex=%{nextThumbStartIndex}&webname=%{webname}&pat=%{pat}',
     liked_artists:        'http://www.pandora.com/content/artistlikes?artistStartIndex=%{nextStartIndex}&webname=%{webname}',
     liked_stations:       'http://www.pandora.com/content/stationlikes?stationStartIndex=%{nextStartIndex}&webname=%{webname}',

data/lib/pandata/parser.rb

@@ -40,68 +40,6 @@ module Pandata
       end
     end
 
-    # @param xml [String]
-    # Returns an array of recent activity names.
-    def get_recent_activity(xml)
-      activity_names = []
-
-      xml_each_item(xml) do |title|
-        activity_names << title
-      end
-
-      activity_names
-    end
-
-    # @param xml [String]
-    # Returns an array of station names.
-    def get_stations(xml)
-      stations = []
-
-      xml_each_item(xml) do |title|
-        stations << title
-      end
-
-      stations
-    end
-
-    # @param xml [String]
-    # @return [String]
-    def get_playing_station(xml)
-      station = ''
-
-      xml_each_item(xml) do |title|
-        station = title  # First title is the station name.
-        break
-      end
-
-      station
-    end
-
-    # @param xml [String]
-    # Returns an array of hashes with :artist and :track keys.
-    def get_bookmarked_tracks(xml)
-      tracks = []
-
-      xml_each_item(xml) do |title|
-        track, artist = title.split(' by ')
-        tracks << { artist: artist, track: track }
-      end
-
-      tracks
-    end
-
-    # @param xml [String]
-    # Returns an array of artist names.
-    def get_bookmarked_artists(xml)
-      artists = []
-
-      xml_each_item(xml) do |title|
-        artists << title
-      end
-
-      artists
-    end
-
     # @param html [String]
     # Returns an array of hashes with :artist and :track keys.
     def get_liked_tracks(html)
@@ -153,16 +91,6 @@ module Pandata
 
     private
 
-    # Loops over each 'item' tag and yields the title and description.
-    # @param xml [String]
-    def xml_each_item(xml)
-      Nokogiri::XML(xml).css('item').each do |item|
-        title = item.at_css('title').text
-        desc = item.at_css('description').text
-        yield(title, desc)
-      end
-    end
-
     # Loops over each .infobox container and yields the title and subtitle.
     # @param html [String]
     def infobox_each_link(html)
@@ -182,8 +110,8 @@ module Pandata
     # @param html [String]
     def doublelink_each_link(html)
       Nokogiri::HTML(html).css('.double-link').each do |doublelink|
-        title_link = doublelink.css('h3 strong').text.strip
-        subtitle_link = doublelink.css('.media--backstageMusic__text div').text.strip
+        title_link = doublelink.css('.media__bd__header').text.strip
+        subtitle_link = doublelink.css('.media__bd__subheader').text.strip
 
         yield(title_link, subtitle_link)
       end

data/lib/pandata/scraper.rb

@@ -41,41 +41,6 @@ module Pandata
       @webname = webname
     end
 
-    # Get the user's recent activity.
-    # @return [Array] array of activity names
-    def recent_activity
-      scrape_for(:recent_activity, :get_recent_activity)
-    end
-
-    # Get the user's playing station.
-    # @return [String]
-    def playing_station
-      scrape_for(:playing_station, :get_playing_station).first
-    end
-
-    # Get the user's stations.
-    # @return [Array] array of station names
-    def stations
-      scrape_for(:stations, :get_stations)
-    end
-
-    # Get the user's bookmarked data.
-    # @param bookmark_type [Symbol]
-    #   - :artists - returns an array of artist names
-    #   - :tracks - returns an array of hashes with :artist and :track keys
-    #   - :all - returns a hash with all bookmarked data
-    def bookmarks(bookmark_type = :all)
-      case bookmark_type
-      when :tracks
-        scrape_for(:bookmarked_tracks, :get_bookmarked_tracks)
-      when :artists
-        scrape_for(:bookmarked_artists, :get_bookmarked_artists)
-      when :all
-        { artists: bookmarks(:artists),
-          tracks: bookmarks(:tracks) }
-      end
-    end
-
     # Get the user's liked data. (The results from giving a 'thumbs up.')
     # @param like_type [Symbol]
     #   - :artists - returns an array of artist names

data/lib/pandata/version.rb

@@ -0,0 +1,10 @@
+module Pandata
+  module Version
+    MAJOR = 2
+    MINOR = 0
+    PATCH = 0
+    BUILD = nil
+
+    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
+  end
+end

metadata

@@ -1,101 +1,100 @@
 --- !ruby/object:Gem::Specification
 name: pandata
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 2.0.0
 platform: ruby
 authors:
 - Brian Ustas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-28 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.3
+        version: 1.6.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.3
+        version: 1.6.7
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.14.0
+        version: 3.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.14.0
+        version: 3.4.0
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 3.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 3.0.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.22.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.13.0
+        version: 1.22.6
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.5
+        version: 0.8.7.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.5
-description: A library and tool for downloading Pandora.com data (likes, bookmarks,
-  stations, etc.)
+        version: 0.8.7.6
+description: A library and tool for downloading Pandora.com data.
 email: brianustas@gmail.com
 executables:
 - pandata
@@ -104,10 +103,6 @@ extra_rdoc_files:
 - LICENSE
 - README.md
 files:
-- LICENSE
-- README.md
-- bin/pandata
-- lib/pandata.rb
 - lib/pandata/argv_parser.rb
 - lib/pandata/cli.rb
 - lib/pandata/data_formatter.rb
@@ -115,6 +110,11 @@ files:
 - lib/pandata/downloader.rb
 - lib/pandata/parser.rb
 - lib/pandata/scraper.rb
+- lib/pandata/version.rb
+- lib/pandata.rb
+- LICENSE
+- README.md
+- bin/pandata
 homepage: https://github.com/ustasb/pandata
 licenses:
 - MIT
@@ -125,17 +125,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.9.1
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: A Pandora.com web scraper