pandata 0.1.2 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b110672f1843569186fff6c0c073defe047b0932
-  data.tar.gz: bf753689e76d89c1b0ce7a92e1c2491f6d021392
+  metadata.gz: 7e2bc9075307ca70cb1e11c34ca8e817d3fea79c
+  data.tar.gz: bca7a313c16dd995cfd0f6094e7cb35b5f539c69
 SHA512:
-  metadata.gz: 0aa2816fa28182cf3bce1f1a813663624279095ff2de56922afa58b110b3e26faa0b661fc1a33df416d90d64c08b3e741174806521d7d65fedf03afa293bb438
-  data.tar.gz: 79cd4ce5b6a65d08585164040d39dd6a952cb8a5fd65ab55023ab4aaf6aaa402e063af7105eb977fd904ee12d6a90ce7bc10d388525c00070840392abac91129
+  metadata.gz: 833e93f00606f5aac3c4a3a297f3fd7ee7f61614de1868b69d9cbbcb68fe218a43275c4087fada150e016d2da02e6bc2ef2dfb08bf368b7a389574140a41f867
+  data.tar.gz: 1ba8b3230c426ec828952376c79e1c4cb649bd3f936db3564ff774f51d0398b1eb9fbbc8beb229a0683ac4dda3e8ff0406cd8266f5a1f5e8af34835afb7347dc

data/README.md

@@ -57,7 +57,7 @@ Next, start scraping!
     # Get all followers
     followers = johns_scraper.followers
 
-For more information, see the documentation for Pandata::Scraper.
+For more information, see the [documentation][2] for Pandata::Scraper.
 
 ### As a Command-Line Tool
 
@@ -82,3 +82,4 @@ For an up-to-date list, check out:
     pandata my_webname --all -o my_pandora_data.txt
 
 [1]: http://www.pandora.com/feeds
+[2]: http://rubydoc.info/gems/pandata/frames

data/bin/pandata

@@ -1,86 +1,8 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/pandata'
-require_relative '../lib/pandata/argv_parser'
-require_relative '../lib/pandata/data_formatter'
+require_relative '../lib/pandata/cli'
 
-options = Pandata::ArgvParser.parse(ARGV)
-
-output_file = options[:output_file]
-if output_file
-  File.delete(output_file) if File.exists?(output_file)
-
-  Object.send(:define_method, :write) do |string|
-    File.open(output_file, 'a') do |file|
-      file.puts string
-    end
-  end
-else
-  def write(string)
-    puts string
-  end
-end
-
-if ARGV.empty?
-  # Print command-line usage help.
-  puts options[:opts]
-  exit
-end
-
-scraper = Pandata::Scraper.get(options[:user_id])
-formatter = Pandata::DataFormatter.new
-
-# If scraper is an array, a Pandora user could not be found with certainty.
-# In this case, scraper will contain webnames similar to options[:user_id].
-if scraper.kind_of?(Array)
-  puts "No exact match for '#{options[:user_id]}'."
-
-  unless scraper.empty?
-    puts "\nWebname results for '#{options[:user_id]}':"
-    puts formatter.list(scraper)
-  end
-
-  exit
-end
-
-scraper_data = {}
-options[:data_to_get].each do |data_type|
-  if /(bookmark|like)e?d_(.*)/ =~ data_type
-    method = $1 << 's'  # 'likes' or 'bookmarks'
-    argument = $2.to_sym  # :tracks, :artists, :stations or :albums
-    scraper_data[data_type] = scraper.public_send(method, argument)
-  else
-    scraper_data[data_type] = scraper.public_send(data_type)
-  end
-end
-
-if options[:return_as_json]
-  require 'json'
-  write JSON.generate(scraper_data)
-  exit
-end
-
-scraper_data.each do |key, value|
-  # Capitalize each word in the key symbol.
-  # e.g. :liked_tracks becomes 'Liked Tracks:'
-  title = key.to_s.split('_').map(&:capitalize).join(' ') << ':'
-
-  if value.empty?
-    output = '  ** No Data **'
-  else
-    output = case key
-             when /playing_station|recent_activity/
-               formatter.list(value)
-             when /liked_tracks|bookmarked_tracks/
-               formatter.tracks(value)
-             when /liked_artists|bookmarked_artists|stations|liked_stations/
-               formatter.sort_list(value)
-             when :liked_albums
-               formatter.albums(value)
-             when /following|followers/
-               formatter.followx(value)
-             end
-  end
-
-  write "#{ title }\n#{ output }"
+begin
+  Pandata::CLI.scrape(ARGV)
+rescue Pandata::PandataError
 end

data/lib/pandata.rb

@@ -6,10 +6,12 @@ require_relative 'pandata/parser'
 require_relative 'pandata/scraper'
 
 module Pandata
+  class PandataError < StandardError; end
+
   module Version
     MAJOR = 0
-    MINOR = 1
-    PATCH = 2
+    MINOR = 2
+    PATCH = 1
     BUILD = nil
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

data/lib/pandata/argv_parser.rb

@@ -15,6 +15,8 @@ module Pandata
     #   - :output_file [String]
     #   - :data_to_get [Array]
     #   - :get_all_data [Boolean]
+    #   - :help [Boolean]
+    #   - :version [Boolean]
     #   - :return_as_json [Boolean]
     def self.parse(argv)
       options = { data_to_get: [] }
@@ -90,20 +92,18 @@ Options:
         end
 
         opts.on_tail("-h", "--help", "Show this message") do
-          puts opts
-          exit
+          options[:help] = true
         end
 
         opts.on_tail("--version", "Show version") do
-          puts Pandata::Version::STRING
-          exit
+          options[:version] = true
         end
       end
 
       options[:opts].parse(argv)
 
       # User ID is the first argument.
-      options[:user_id] = argv.shift
+      options[:user_id] = argv[0]
 
       if get_all_data
         options[:data_to_get] = [

data/lib/pandata/cli.rb

@@ -0,0 +1,144 @@
+require 'json'
+require 'ruby-progressbar'
+require_relative '../pandata'
+require_relative 'argv_parser'
+require_relative 'data_formatter'
+
+module Pandata
+
+  # Pandata command-line interface
+  class CLI
+
+    def self.scrape(argv)
+      options = Pandata::ArgvParser.parse(argv)
+
+      if argv.empty? || options[:help]
+        puts options[:opts].to_s  # Log usage information
+      elsif options[:version]
+        puts Pandata::Version::STRING
+      else
+        new(options).download_and_output
+      end
+    end
+
+    def initialize(options)
+      @data_to_get = options[:data_to_get]
+      @output_file = options[:output_file]
+      @return_as_json = options[:return_as_json]
+
+      @scraper = scraper_for(options[:user_id])
+      @scraper.download_cb = method(:update_progress)
+    end
+
+    def update_progress(num_data)
+      progressbar.progress += num_data
+    end
+
+    def download_and_output
+      output_data format_data(download_data, @return_as_json)
+    end
+
+    private
+
+    def progressbar
+      @progressbar ||= ProgressBar.create(
+        title: 'Data Downloaded',
+        format: '%t: %c',
+        total: nil
+      )
+    end
+
+    def formatter
+      @formatter ||= DataFormatter.new
+    end
+
+    def log(msg)
+      puts msg
+    end
+
+    # Writes the data to STDOUT or a file.
+    # @param formatted_data [String]
+    def output_data(formatted_data)
+      @progressbar.stop if @progressbar
+
+      if @output_file
+        File.write(@output_file, formatted_data)
+      else
+        log formatted_data
+      end
+    end
+
+    # Formats data as a string list or JSON.
+    # @param data [Hash]
+    # @param json [Boolean]
+    # @return [String]
+    def format_data(data, json = false)
+      if json
+        JSON.generate(data)
+      else
+        data.map do |category, cat_data|
+          # Capitalize each word in the category symbol.
+          # e.g. :liked_tracks becomes 'Liked Tracks'
+          title = category.to_s.split('_').map(&:capitalize).join(' ')
+
+          output = if cat_data.empty?
+                     "  ** No Data **\n"
+                   else
+                     case category
+                     when /playing_station|recent_activity/
+                       formatter.list(cat_data)
+                     when /liked_tracks|bookmarked_tracks/
+                       formatter.tracks(cat_data)
+                     when /liked_artists|bookmarked_artists|stations|liked_stations/
+                       formatter.sort_list(cat_data)
+                     when :liked_albums
+                       formatter.albums(cat_data)
+                     when /following|followers/
+                       formatter.followx(cat_data)
+                     end
+                   end
+
+          "#{title}:\n#{output}"
+        end.join
+      end
+    end
+
+    # Downloads the user's desired data.
+    # @return [Hash]
+    def download_data
+      scraper_data = {}
+
+      @data_to_get.each do |data_category|
+        if /(bookmark|like)e?d_(.*)/ =~ data_category
+          method = $1 << 's'  # 'likes' or 'bookmarks'
+          argument = $2.to_sym  # :tracks, :artists, :stations or :albums
+          scraper_data[data_category] = @scraper.public_send(method, argument)
+        else
+          scraper_data[data_category] = @scraper.public_send(data_category)
+        end
+      end
+
+      scraper_data
+    end
+
+    # Returns a scraper for the user's id.
+    # @param user_id [String] webname or email
+    # @return [Pandata::Scraper]
+    def scraper_for(user_id)
+      scraper = Pandata::Scraper.get(user_id)
+
+      if scraper.kind_of?(Array)
+        log "No exact match for '#{user_id}'."
+
+        unless scraper.empty?
+          log "\nWebname results for '#{user_id}':\n#{formatter.list(scraper)}"
+        end
+
+        raise PandataError, "Could not create a scraper for '#{user_id}'."
+      end
+
+      scraper
+    end
+
+  end
+end

data/lib/pandata/data_urls.rb

@@ -1,6 +1,6 @@
 module Pandata
   # Number of results to get from a feeds.pandora.com URL.
-  MAX_RESULTS = 100000  # Get everything...
+  MAX_RESULTS = 100_000  # Get everything...
 
   # URLs to Pandora's data!
   DATA_FEED_URLS = {

data/lib/pandata/downloader.rb

@@ -1,30 +1,23 @@
 require 'json'
 require 'open-uri'
+require_relative '../pandata'
 
 module Pandata
-  class PandataError < StandardError; end
 
-  # Retrieves data from Pandora.com and handles errors.
+  # Retrieves data from Pandora.com and handles network errors.
   class Downloader
+
     # A GitHub Gist that contains an updated cookie allowing access to 'login-only' visible data.
     CONFIG_URL = 'https://gist.github.com/ustasb/596f1ee96d03463fde77/raw/pandata_config.json'
 
-    class << self
-      attr_accessor :cookie
-    end
-
-    # Gets a Pandora cookie and returns a Downloader instance.
-    def initialize
-      unless Downloader.cookie
-        Downloader.cookie = get_cookie
-      end
-    end
+    # The cached cookie.
+    @@cookie = nil
 
     # Downloads and reads a page from a URL.
     # @param url [String]
     # @return [String] contents of page
-    def read_page(url)
-      download(url, Downloader.cookie).read
+    def self.read_page(url)
+      download(url, get_cookie).read
     end
 
     private
@@ -33,19 +26,21 @@ module Pandata
     # @param url [String]
     # @param cookie [String]
     # @return [File]
-    def download(url, cookie = '')
+    def self.download(url, cookie = '')
       escaped_url = URI.escape(url)
 
-      begin
-        open(escaped_url, 'Cookie' => cookie, :read_timeout => 5)
-      rescue OpenURI::HTTPError => error
-        puts "The network request for:\n  #{url}\nreturned an error:\n  #{error.message}"
-        puts "Please try again later or update Pandata. Sorry about that!\n\nFull error:"
-        raise PandataError
-      end
+      open(escaped_url, 'Cookie' => cookie, :read_timeout => 5)
+    rescue OpenURI::HTTPError => error
+      puts "The network request for:\n  #{url}\nreturned an error:\n  #{error.message}"
+      puts "Please try again later or update Pandata. Sorry about that!\n\nFull error:"
+      raise PandataError
+    end
+
+    def self.get_cookie
+      @@cookie ||= download_cookie
     end
 
-    def get_cookie
+    def self.download_cookie
       config = JSON.parse download(CONFIG_URL).read
 
       if Gem::Version.new(Pandata::Version::STRING) <= Gem::Version.new(config['required_update_for'])
@@ -54,5 +49,6 @@ module Pandata
 
       config['cookie']
     end
+
   end
 end

data/lib/pandata/scraper.rb

@@ -12,6 +12,9 @@ module Pandata
     # the user ties a new email address to their Pandora account.
     attr_reader :webname
 
+    # A Proc that gets called after some data has been downloaded.
+    attr_accessor :download_cb
+
     # If possible, get a Scraper instance for the user_id otherwise return
     # an array of similar webnames.
     # @param user_id [String] email or webname
@@ -19,7 +22,7 @@ module Pandata
     # @return [Array] array of similar webnames
     def self.get(user_id)
       search_url = DATA_FEED_URLS[:user_search] % { searchString: user_id }
-      html = Downloader.new.read_page(search_url)
+      html = Downloader.read_page(search_url)
       webnames = Parser.new.get_webnames_from_search(html)
 
       if webnames.include?(user_id)
@@ -34,7 +37,6 @@ module Pandata
 
     private_class_method :new
     def initialize(webname)
-      @downloader = Downloader.new
       @parser = Parser.new
       @webname = webname
     end
@@ -134,6 +136,8 @@ module Pandata
           results.push(new_data)
         end
 
+        @download_cb[new_data.size] if @download_cb
+
         get_url(data_type, next_data_indices) if next_data_indices
       end
 
@@ -149,7 +153,7 @@ module Pandata
       next_data_indices = {}
 
       while next_data_indices
-        html = @downloader.read_page(url)
+        html = Downloader.read_page(url)
         next_data_indices = @parser.get_next_data_indices(html)
         url = yield(html, next_data_indices)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pandata
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.1
 platform: ruby
 authors:
 - Brian Ustas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-18 00:00:00.000000000 Z
+date: 2013-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,20 +24,62 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.5.6
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.2
+        version: 2.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.0
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.12.2
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -63,6 +105,7 @@ extra_rdoc_files:
 - README.md
 files:
 - lib/pandata/argv_parser.rb
+- lib/pandata/cli.rb
 - lib/pandata/data_formatter.rb
 - lib/pandata/data_urls.rb
 - lib/pandata/downloader.rb
@@ -92,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: A Pandora.com web scraper