bnet_scraper 0.3.1 → 0.4.0

data/lib/bnet_scraper/starcraft2.rb

@@ -94,15 +94,14 @@ module BnetScraper
     #   scraped from the website
     def self.full_profile_scrape bnet_id, account, region = 'na'
       profile_scraper = ProfileScraper.new bnet_id: bnet_id, account: account, region: region
-      profile_output  = profile_scraper.scrape
-
-      parsed_leagues = []
-      profile_output[:leagues].each do |league|
-        league_scraper = LeagueScraper.new url: league[:href]
-        parsed_leagues << league_scraper.scrape
+      profile = profile_scraper.scrape
+      profile.leagues.each do |league|
+        league.scrape_league
       end
-      profile_output[:leagues] = parsed_leagues
-      return profile_output
+      profile.achievements
+      profile.match_history
+      
+      return profile
     end
 
     # Determine if Supplied profile is valid.  Useful for validating now before an 

data/lib/bnet_scraper/starcraft2/achievement_scraper.rb

@@ -72,11 +72,11 @@ module BnetScraper
       def scrape_progress
         progress_ach = response.css("#progress-module .achievements-progress:nth(2) span")
         @progress = {
-          liberty_campaign: progress_ach[0].inner_text,
-          exploration:      progress_ach[1].inner_text,
-          custom_game:      progress_ach[2].inner_text,
-          cooperative:      progress_ach[3].inner_text,
-          quick_match:      progress_ach[4].inner_text,
+          liberty_campaign: response.css(".progress-tile:nth-child(1) .profile-progress span").inner_text,
+          exploration:      response.css(".progress-tile:nth-child(2) .profile-progress span").inner_text,
+          custom_game:      response.css(".progress-tile:nth-child(3) .profile-progress span").inner_text,
+          cooperative:      response.css(".progress-tile:nth-child(4) .profile-progress span").inner_text,
+          quick_match:      response.css(".progress-tile:nth-child(5) .profile-progress span").inner_text,
         }
       end
 

data/lib/bnet_scraper/starcraft2/league.rb

@@ -0,0 +1,54 @@
+module BnetScraper
+  module Starcraft2
+    class League
+      attr_accessor :id, :name, :href, :season, :name, :division, :size, :random, :bnet_id,
+        :account
+
+      def initialize options = {}
+        options.each_key do |key|
+          self.send "#{key}=", options[key]
+        end
+      end
+
+      def name
+        scrape_or_return :@name
+      end
+
+      def season
+        scrape_or_return :@season
+      end
+
+      def division
+        scrape_or_return :@division
+      end
+
+      def size
+        scrape_or_return :@size
+      end
+
+      def bnet_id
+        scrape_or_return :@bnet_id
+      end
+
+      def account
+        scrape_or_return :@account
+      end
+
+      def scrape_or_return attribute
+        if self.instance_variable_get(attribute)
+          return self.instance_variable_get(attribute)
+        else
+          scrape_league
+          self.instance_variable_get(attribute)
+        end
+      end
+
+      def scrape_league
+        scraped_data = LeagueScraper.new(url: href).scrape
+        scraped_data.each_key do |key|
+          self.send "#{key}=", scraped_data[key]
+        end
+      end
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/league_scraper.rb

@@ -1,3 +1,5 @@
+require 'bnet_scraper/starcraft2/league'
+
 module BnetScraper
   module Starcraft2
     # This pulls information on a specific league for a specific account.  It is best used either in conjunction with a
@@ -34,7 +36,7 @@ module BnetScraper
         if @response.success?
           @response = Nokogiri::HTML(@response.body)
           value = @response.css(".data-title .data-label h3").inner_text().strip 
-          header_regex = /Season (\d{1}) - \s+(\dv\d)( Random)? (\w+)\s+Division (.+)/
+          header_regex = /(.+) -\s+(\dv\d)( Random)? (\w+)\s+Division (.+)/
           header_values = value.match(header_regex).to_a
           header_values.shift()
           @season, @size, @random, @division, @name = header_values

data/lib/bnet_scraper/starcraft2/match.rb

@@ -0,0 +1,15 @@
+module BnetScraper
+  module Starcraft2
+    class Match
+      attr_accessor :outcome, :map_name, :type, :date
+
+      def won?
+        @outcome == :won
+      end
+
+      def lost?
+        @outcome == :loss
+      end
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/match_history_scraper.rb

@@ -1,3 +1,5 @@
+require 'bnet_scraper/starcraft2/match'
+
 module BnetScraper
   module Starcraft2
     # This pulls the 25 most recent matches played for an account. Note that this is only as up-to-date as battle.net is, and
@@ -42,29 +44,18 @@ module BnetScraper
 
         response.css('.match-row').each do |m|
           match = {}
+          match = Match.new
 
           cells = m.css('td')
-          match[:map_name] = cells[1].inner_text
-          match[:type] = cells[2].inner_text
-          match[:outcome] = (cells[3].inner_text.strip == 'Win' ? :win : :loss)
-          match[:date] = cells[4].inner_text.strip
+          match.map_name = cells[1].inner_text
+          match.type = cells[2].inner_text
+          match.outcome = (cells.css('.match-loss') ? :win : :loss)
+          match.date = cells[4].inner_text.strip
+
           @matches << match 
-          if match[:outcome] == :win
-            @wins += 1
-          else
-            @losses += 1
-          end
-          output
         end
 
-      end
-
-      def output
-        {
-          matches:  @matches,
-          wins:     @wins,
-          losses:   @losses 
-        }
+        @matches
       end
     end
   end

data/lib/bnet_scraper/starcraft2/profile.rb

@@ -0,0 +1,45 @@
+
+module BnetScraper
+  module Starcraft2
+    class Profile
+      attr_accessor :portrait, :url, :achievement_points, :current_solo_league, 
+        :highest_solo_league, :current_team_league, :highest_team_league,
+        :career_games, :games_this_season, :terran_swarm_level, :protoss_swarm_level,
+        :zerg_swarm_level, :leagues, :swarm_levels
+
+      def initialize options = {}
+        options.each_key do |key|
+          self.send "#{key}=", options[key]
+        end
+      end
+
+      def achievements
+        @achievements ||= AchievementScraper.new(url: url).scrape
+      end
+
+      def recent_achievements
+        achievements[:recent]
+      end
+
+      def progress_achievements
+        achievements[:progress]
+      end
+
+      def showcase_achievements
+        achievements[:showcase]
+      end
+
+      def match_history
+        @match_history ||= MatchHistoryScraper.new(url: url).scrape
+      end
+
+      def swarm_levels
+        {
+          zerg: @zerg_swarm_level,
+          protoss: @protoss_swarm_level,
+          terran: @terran_swarm_level
+        }
+      end
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/profile_scraper.rb

@@ -1,3 +1,5 @@
+require 'bnet_scraper/starcraft2/profile'
+
 module BnetScraper
   module Starcraft2
     # This pulls basic profile information for an account, as well as an array of league URLs.  This is a good starting
@@ -28,19 +30,22 @@ module BnetScraper
     #     ]
     #   }
     class ProfileScraper < BaseScraper
-      attr_reader :achievement_points, :career_games, :race, :leagues, :most_played,
-        :games_this_season, :highest_solo_league, :current_solo_league, :highest_team_league,
-        :current_team_league, :portrait
+      attr_reader :achievement_points, :career_games, :leagues, :games_this_season, 
+        :highest_solo_league, :current_solo_league, :highest_team_league,
+        :current_team_league, :portrait, :terran_swarm_level, :protoss_swarm_level,
+        :zerg_swarm_level, :profile
 
       def initialize options = {}
         super
         @leagues = []
+        @profile ||= Profile.new url: profile_url
       end
 
       def scrape
         get_profile_data
         get_league_list
-        output
+
+        @profile
       end
 
       # scrapes the profile page for basic account information
@@ -50,64 +55,83 @@ module BnetScraper
         if response.success?
           html = Nokogiri::HTML(response.body)
 
-          # Portraits use spritemaps, so we extract positions and map to 
-          # PORTRAITS.
-          @portrait = begin
-            portrait = html.css("#profile-header #portrait span").attr('style').to_s.scan(/url\('(.*?)'\) ([\-\d]+)px ([\-\d]+)px/).flatten
-            portrait_map, portrait_size = portrait[0].scan(/(\d)\-(\d+)\.jpg/)[0]
-            portrait_position = (((0-portrait[2].to_i) / portrait_size.to_i) * 6) + ((0-portrait[1].to_i) / portrait_size.to_i + 1)
-            PORTRAITS[portrait_map.to_i][portrait_position-1]
-          rescue 
-            nil
-          end
+          @profile.achievement_points = html.css("#profile-header h3").inner_html()
+          @profile.career_games = html.css(".career-stat-block:nth-child(4) .stat-value").inner_html()
+          @profile.games_this_season = html.css(".career-stat-block:nth-child(5) .stat-value").inner_html()
+
+          get_portrait html
+          get_solo_league_info html
+          get_team_league_info html
+          get_swarm_levels html
+        else
+          raise BnetScraper::InvalidProfileError
+        end
+      end
+
+      def get_portrait html
+        # Portraits use spritemaps, so we extract positions and map to 
+        # PORTRAITS.
+        @profile.portrait = begin
+          portrait = html.css("#profile-header #portrait span").attr('style').to_s.scan(/url\('(.*?)'\) ([\-\d]+)px ([\-\d]+)px/).flatten
+          portrait_map, portrait_size = portrait[0].scan(/(\d)\-(\d+)\.jpg/)[0]
+          portrait_position = (((0-portrait[2].to_i) / portrait_size.to_i) * 6) + ((0-portrait[1].to_i) / portrait_size.to_i + 1)
+          PORTRAITS[portrait_map.to_i][portrait_position-1]
+        rescue 
+          nil
+        end
+      end
 
-          @race = html.css(".stat-block:nth-child(4) h2").inner_html()
-          @achievement_points = html.css("#profile-header h3").inner_html()
-          @career_games = html.css(".stat-block:nth-child(3) h2").inner_html()
-          @most_played = html.css(".stat-block:nth-child(2) h2").inner_html()
-          @games_this_season = html.css(".stat-block:nth-child(1) h2").inner_html()
-
-          if html.css("#best-finish-SOLO div")[0]
-            @highest_solo_league = html.css("#best-finish-SOLO div")[0].children[2].inner_text.strip
-            if html.css("#best-finish-SOLO div")[0].children[8]
-              @current_solo_league = html.css("#best-finish-SOLO div")[0].children[8].inner_text.strip
-            else
-              @current_solo_league = html.css("#best-finish-SOLO div")[0].children[5].inner_text.strip
-            end
+      def get_solo_league_info html
+        if html.css("#best-finish-SOLO div")[0]
+          @profile.highest_solo_league = html.css("#best-finish-SOLO div")[0].children[2].inner_text.strip
+          if html.css("#best-finish-SOLO div")[0].children[8]
+            @profile.current_solo_league = html.css("#best-finish-SOLO div")[0].children[8].inner_text.strip
           else
-            @highest_solo_league = "Not Yet Ranked"
-            @current_solo_league = "Not Yet Ranked"
+            @profile.current_solo_league = "Not Yet Ranked"
           end
+        else
+          @profile.highest_solo_league = "Not Yet Ranked"
+          @profile.current_solo_league = "Not Yet Ranked"
+        end
+      end
 
-          if html.css("#best-finish-TEAM div")[0] 
-            @highest_team_league = html.css("#best-finish-TEAM div")[0].children[2].inner_text.strip
-            if html.css("#best-finish-TEAM div")[0].children[8]
-              @current_team_league = html.css("#best-finish-TEAM div")[0].children[8].inner_text.strip
-            else
-              @current_team_league = html.css("#best-finish-TEAM div")[0].children[5].inner_text.strip
-            end
+      def get_team_league_info html
+        if html.css("#best-finish-TEAM div")[0] 
+          @profile.highest_team_league = html.css("#best-finish-TEAM div")[0].children[2].inner_text.strip
+          if html.css("#best-finish-TEAM div")[0].children[8]
+            @profile.current_team_league = html.css("#best-finish-TEAM div")[0].children[8].inner_text.strip
           else
-            @highest_team_league = "Not Yet Ranked"
-            @current_team_league = "Not Yet Ranked"
+            @profile.current_team_league = "Not Yet Ranked"
           end
-
         else
-          raise BnetScraper::InvalidProfileError
+          @profile.highest_team_league = "Not Yet Ranked"
+          @profile.current_team_league = "Not Yet Ranked"
         end
       end
 
+      def get_swarm_levels html
+        @profile.protoss_swarm_level = get_swarm_level :protoss, html
+        @profile.terran_swarm_level = get_swarm_level :terran, html
+        @profile.zerg_swarm_level = get_swarm_level :zerg, html
+      end
+
+      def get_swarm_level race, html
+        level = html.css(".race-level-block.#{race} .level-value").inner_html
+        level.match(/Level (\d+)/)[1].to_i
+      end
+
       # scrapes the league list from account's league page and sets an array of URLs
       def get_league_list
         response = Faraday.get profile_url + "ladder/leagues"
         if response.success?
           html = Nokogiri::HTML(response.body)
 
-          @leagues = html.css("a[href*='#current-rank']").map do |league|
-            {
-              name: league.inner_html().strip,
+          @profile.leagues = html.css("a[href*='#current-rank']").map do |league|
+            League.new({
+              name: league.inner_text().strip,
               id: league.attr('href').sub('#current-rank',''),
               href: "#{profile_url}ladder/#{league.attr('href')}"
-            }
+            })
           end
         else
           raise BnetScraper::InvalidProfileError

data/spec/spec_helper.rb

@@ -1,5 +1,19 @@
 $:.push File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'bnet_scraper'
+require 'pry'
+require 'vcr'
 Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each {|f| require f }
 
+VCR.configure do |config|
+  config.cassette_library_dir = 'fixtures/vcr_cassettes'
+  config.hook_into :fakeweb
+end
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.around(:each, :vcr) do |example|
+     name = example.metadata[:full_description].split(/\s+/, 2).join("/").underscore.gsub(/[^\w\/]+/, "_")
+     VCR.use_cassette(name) { example.call }
+  end
+end

data/spec/starcraft2/achievement_scraper_spec.rb

@@ -9,97 +9,81 @@ describe BnetScraper::Starcraft2::AchievementScraper do
     let(:subject) { scraper_class.new(url: url) }
   end
 
-  describe '#get_response' do
-    it 'should get the HTML response to be scraped' do
-      subject.response.should be_nil
-      subject.get_response
-      subject.response.should_not be_nil
-    end
-  end
-
-  describe '#scrape' do
-    it 'should call get_response and trigger scraper methods' do
-      subject.should_receive(:get_response)
-      subject.should_receive(:scrape_progress)
-      subject.should_receive(:scrape_recent)
-      subject.should_receive(:scrape_showcase)
-      subject.scrape
-    end
-
+  describe 'scrape' do
     it 'should return InvalidProfileError if response is 404' do
-      url = 'http://us.battle.net/sc2/en/profile/2377239/1/SomeDude/achievements/'
-      scraper = BnetScraper::Starcraft2::AchievementScraper.new(url: url)
-      expect { scraper.scrape }.to raise_error(BnetScraper::InvalidProfileError)
-    end
-  end
-
-  describe '#scrape_showcase' do
-    before :each do
-      subject.get_response
-      subject.scrape_showcase
-    end
-
-    it 'should set the showcase' do
-      subject.showcase.should have(5).achievements
-    end
-  end
-
-  describe '#scrape_recent' do
-    before :each do
-      subject.get_response
-      subject.scrape_recent
-    end
-
-    it 'should have the title of the achievement' do
-      subject.recent[0][:title].should == 'Blink of an Eye'  
-    end
-
-    it 'should have the description of the achievement' do
-      # this is a cop-out because the string contains UTF-8. Please fix this. - Cad
-      subject.recent[0][:description].should be_a String 
-    end
-
-    it 'should have the date the achievement was earned' do
-      subject.recent[0][:earned].should == '3/5/2012' 
-    end
-  end
-
-  describe '#scrape_progress' do
-    before :each do
-      subject.get_response
-      subject.scrape_progress
-    end 
-
-    it 'should set the liberty campaign progress' do
-      subject.progress[:liberty_campaign].should == '1580'
-    end
-
-    it 'should set the exploration progress' do
-      subject.progress[:exploration].should == '480'
-    end
-
-    it 'should set the custom game progress' do
-      subject.progress[:custom_game].should == '330'
-    end
-
-    it 'should set the cooperative progress' do
-      subject.progress[:cooperative].should == '660'
+      VCR.use_cassette('invalid_achievement') do
+        url = 'http://us.battle.net/sc2/en/profile/2377239/1/SomeDude/achievements/'
+        scraper = BnetScraper::Starcraft2::AchievementScraper.new(url: url)
+        expect { scraper.scrape }.to raise_error(BnetScraper::InvalidProfileError)
+      end
     end
 
-    it 'should set the quick match progress' do
-      subject.progress[:quick_match].should == '170'
+    context 'valid' do
+      before do
+        VCR.use_cassette('demon_achievements') do
+          subject.get_response
+        end
+      end
+
+      describe 'showcase' do
+        before { subject.scrape_showcase }
+        its(:showcase) { should have(5).achievements }
+      end
+
+      describe 'recent' do
+        before { subject.scrape_recent }
+
+        it 'should have the title of the achievement' do
+          subject.recent[0][:title].should == 'Three-way Dominant'  
+        end
+
+        it 'should have the description of the achievement' do
+          # this is a cop-out because the string contains UTF-8. Please fix this. - Cad
+          subject.recent[0][:description].should be_a String 
+        end
+
+        it 'should have the date the achievement was earned' do
+          subject.recent[0][:earned].should == '2/7/2013' 
+        end
+      end
+
+      describe 'progress' do
+        before { subject.scrape_progress }
+
+        it 'should set the liberty campaign progress' do
+          subject.progress[:liberty_campaign].should == '1580'
+        end
+
+        it 'should set the exploration progress' do
+          subject.progress[:exploration].should == '0'
+        end
+
+        it 'should set the custom game progress' do
+          subject.progress[:custom_game].should == '1280'
+        end
+
+        it 'should set the cooperative progress' do
+          subject.progress[:cooperative].should == '120'
+        end
+
+        it 'should set the quick match progress' do
+          subject.progress[:quick_match].should == '220'
+        end
+      end
     end
   end
 
   describe '#output' do
     it 'should return the scraped data when scrape has been called' do
-      subject.scrape
-      expected = {
-        recent: subject.recent,
-        showcase: subject.showcase,
-        progress: subject.progress
-      }
-      subject.output.should == expected
+      VCR.use_cassette('demon_achievements') do
+        subject.scrape
+        expected = {
+          recent: subject.recent,
+          showcase: subject.showcase,
+          progress: subject.progress
+        }
+        subject.output.should == expected
+      end
     end
   end
 end