bnet_scraper 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc
+doc/*
+.yardoc/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format nested

data/.travis.yml

@@ -0,0 +1,4 @@
+notifications:
+  email:
+    recipients:
+      - anordman@majorleaguegaming.com

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in bnet_scraper.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Andrew Nordman
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,136 @@
+# BnetScraper
+
+BnetScraper is a Nokogiri-based scraper of Battle.net profile information.  Currently this only includes Starcraft2.
+
+# Installation
+
+Run `gem install bnet_scraper` or add `gem 'bnet_scraper'` to your `Gemfile`.
+
+# Usage
+
+All of the scrapers take an options hash, and can be created by either passing a URL string for the profile URL or
+passing the account information in the options hash.  Thus, either of these two approaches work:
+
+``` ruby
+BnetScraper::Starcraft2::ProfileScraper.new(url: 'http://us.battle.net/sc2/en/profile/12345/1/TestAccount/')
+BnetScraper::Starcraft2::ProfileScraper.new(bnet_id: '12345', account: 'TestAccount', region: 'na')
+```
+
+There are several scrapers that pull various information.  They are:
+
+* BnetScraper::Starcraft2::ProfileScraper - collects basic profile information and an array of league URLs
+* BnetScraper::Starcraft2::LeagueScraper - collects data on a particular league for a particular Battle.net account
+* BnetScraper::Starcraft2::AchievementScraper - collects achievement data for the account.
+* BnetScraper::Starcraft2::MatchHistoryScraper - collects the 25 most recent matches played on the account
+
+All scrapers have a `#scrape` method that triggers the scraping and storage.  By default they will return the result,
+but an additional `#output` method exists to retrieve the results subsequent times without re-scraping.
+
+## BnetScraper::Starcraft2::ProfileScraper
+
+This pulls basic profile information for an account, as well as an array of league URLs.  This is a good starting
+point for league scraping as it provides the league URLs necessary to do supplemental scraping.
+
+``` ruby
+scraper = BnetScraper::Starcraft2::ProfileScraper.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/')
+scraper.scrape
+# => {
+  bnet_id: '2377239',
+  account: 'Demon',
+  bnet_index: 1,
+  race: 'Protoss',
+  wins: '684',
+  achievement_points: '3630',
+  leagues: [
+    {
+      name: "1v1 Platinum Rank 95",
+      id:   "96905",
+      href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96905#current-rank"
+    }
+  ]
+}
+```
+
+## BnetScraper::Starcraft2::LeagueScraper
+
+This pulls information on a specific league for a specific account.  It is best used either in conjunction with a
+profile scrape that profiles a URL, or if you happen to know the specific league\_id and can pass it as an option.
+
+``` ruby
+scraper = BnetScraper::Starcraft2::LeagueScraper.new(league_id: '12345', account: 'Demon', bnet_id: '2377239')
+scraper.scrape
+# => {
+  season: '6',
+  name: 'Aleksander Pepper',
+  division: 'Diamond',
+  size: '4v4',
+  random: false,
+  bnet_id: '2377239',
+  account: 'Demon'
+}
+```
+
+## BnetScraper::Starcraft2::AchievementScraper
+
+This pulls achievement information for an account.  Note that currently only returns the overall achievements,
+not the in-depth, by-category achievement information.
+
+``` ruby
+scraper = BnetScraper::Starcraft2::AchievementScraper.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/')
+scraper.scrape
+# => {
+  recent: [
+    { title: 'Blink of an Eye', description: 'Complete round 24 in "Starcraft Master" without losing any stalkers', earned: '3/5/2012' },
+    { title: 'Whack-a-Roach', description: 'Complete round 9 in "Starcraft Master" in under 45 seconds', earned: '3/5/2012' },
+    { title: 'Safe Zone', description: 'Complete round 8 in "Starcraft Master" without losing any stalkers', earned: '3/5/2012' },
+    { title: 'Starcraft Master', description: 'Complete all 30 rounds in "Starcraft Master"', earned: '3/5/2012' },
+    { title: 'Starcraft Expert', description: 'Complete any 25 rounds in "Starcraft Master"', earned: '3/5/2012' },
+    { title: 'Starcraft Apprentice', description: 'Complete any 20 rounds in "Starcraft Master"', earned: '3/5/2012' }
+  ],
+  showcase: [
+    { title: 'Hot Shot', description: 'Finish a Qualification Round with an undefeated record.' },
+    { title: 'Starcraft Master', description: 'Complete all rounds in "Starcraft Master"' },
+    { title: 'Team Protoss 500', description: 'Win 500 team league matches as Protoss' },
+    { title: 'Night of the Living III', description: 'Survive 15 Infested Horde Attacks in the "Night 2 Die" mode of the "Left 2 Die" scenario.' },
+    { title: 'Team Top 100 Diamond', description: 'Finish a Season in Team Diamond Division' }
+												
+  ],
+  progress: {
+    liberty_campaign: '1580',
+    exploration: '480',
+    custom_game: '330',
+    cooperative: '660',
+    quick_match: '170'
+  }
+}
+```
+
+## BnetScraper::Starcraft2::MatchHistoryScraper
+
+This pulls the 25 most recent matches played for an account. Note that this is only as up-to-date as battle.net is, and
+will likely not be as fast as in-game.
+
+``` ruby
+scraper = BnetScraper::Starcraft2::MatchHistoryScraper.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/')
+scraper.scrape
+# => {
+  wins: '15',
+  losses: '10',
+  matches: [
+    { map_name: 'Bx Monobattle - Sand Canyon (Fix)', outcome: :win, type: 'Custom', date: '3/12/2012' },
+    { map_name: 'Deadlock Ridge', outcome: :loss, type: '4v4', date: '3/12/2012' },
+    { map_name: 'District 10', outcome: :win, type: '4v4', date: '3/12/2012' },
+    # ...
+  ]
+}
+```
+
+# Contribute!
+
+I would love to see contributions!  Please send a pull request with a feature branch containing specs 
+(Chances are excellent I will break it if you do not) and I will take a look.  Please do not change the version
+as I tend to bundle multiple fixes together before releasing a new version anyway.
+
+# Author
+
+Written by [Andrew Nordman](http://github.com/cadwallion), see LICENSE for details.

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rspec_opts = ['--backtrace']
+  # spec.ruby_opts = ['-w']
+end
+
+task :default => :spec

data/benchmarks/sc2_profile_parsing.rb

@@ -0,0 +1,24 @@
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'benchmark'
+require 'bnet_scraper'
+
+@@bnet_accounts = [
+  { bnet_id: 2377239, name: 'Demon', race: 'protoss' },
+  { bnet_id: 2035618, name: 'Mykaelos', race: 'random' },
+  { bnet_id: 1826063, name: 'Fyrefly', race: 'zerg' },
+  { bnet_id: 2539344, name: 'Cadwallion', race: 'terran' }
+]
+
+def parse_random_account
+  account = @@bnet_accounts[rand(@@bnet_accounts.size)] 
+  BnetScraper::Starcraft2::ProfileScraper.new(account[:bnet_id], account[:name]).scrape
+end
+
+Benchmark.bmbm(7) do |x|
+  x.report('1') { parse_random_account }
+  x.report('5') { 5.times { parse_random_account } }
+  x.report('10') { 10.times { parse_random_account } }
+  x.report('25') { 25.times { parse_random_account } }
+  x.report('50') { 50.times { parse_random_account } }
+end

data/bnet_scraper.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "bnet_scraper"
+  s.version     = "0.0.1"
+  s.authors     = ["Andrew Nordman"]
+  s.email       = ["anordman@majorleaguegaming.com"]
+  s.homepage    = "https://github.com/agoragames/bnet_scraper/"
+  s.summary     = %q{Battle.net Profile Scraper}
+  s.description = %q{BnetScraper is a Nokogiri-based scraper of Battle.net profile information. Currently this only includes Starcraft2.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'nokogiri'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'fakeweb'
+end

data/lib/bnet_scraper.rb

@@ -0,0 +1,7 @@
+require 'bnet_scraper/starcraft2'
+require 'net/http'
+require 'open-uri'
+require 'nokogiri'
+
+module BnetScraper
+end

data/lib/bnet_scraper/starcraft2.rb

@@ -0,0 +1,46 @@
+require 'bnet_scraper/starcraft2/base_scraper'
+require 'bnet_scraper/starcraft2/profile_scraper'
+require 'bnet_scraper/starcraft2/league_scraper'
+require 'bnet_scraper/starcraft2/achievement_scraper'
+require 'bnet_scraper/starcraft2/match_history_scraper'
+
+module BnetScraper
+  # This module contains everything about scraping Starcraft 2 Battle.net accounts.
+  # See `BnetScraper::Starcraft2::ProfileScraper` and `BnetScraper::Starcraft2::LeagueScraper`
+  # for more details
+  module Starcraft2
+    REGIONS = {
+      'na'  => { domain: 'us.battle.net', dir: 'en' },
+      'eu'  => { domain: 'eu.battle.net', dir: 'eu' },
+      'cn'  => { domain: 'www.battlenet.com.cn', dir: 'zh' },
+      'sea' => { domain: 'sea.battle.net', dir: 'en' },
+      'fea' => { domain: 'tw.battle.net', dir: 'zh' } 
+    }
+
+    # This is a convenience method that chains calls to ProfileScraper,
+    # followed by a scrape of each league returned in the `leagues` array
+    # in the profile_data.  The end result is a fully scraped profile with
+    # profile and league data in a hash.
+    #
+    # See `BnetScraper::Starcraft2::ProfileScraper` for more information on
+    # the parameters being sent to `#full_profile_scrape`.
+    #
+    # @param bnet_id - Battle.net Account ID 
+    # @param account - Battle.net Account Name
+    # @param region  - Battle.net Account Region
+    # @return profile_data - Hash containing complete profile and league data
+    #   scraped from the website
+    def self.full_profile_scrape bnet_id, account, region = 'na'
+      profile_scraper = ProfileScraper.new bnet_id: bnet_id, account: account, region: region
+      profile_output  = profile_scraper.scrape
+
+      parsed_leagues = []
+      profile_output[:leagues].each do |league|
+        league_scraper = LeagueScraper.new url: league[:href]
+        parsed_leagues << league_scraper.scrape
+      end
+      profile_output[:leagues] = parsed_leagues
+      return profile_output
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/achievement_scraper.rb

@@ -0,0 +1,63 @@
+module BnetScraper
+  module Starcraft2
+    class AchievementScraper < BaseScraper
+      attr_reader :recent, :progress, :showcase, :response
+
+      def scrape
+        get_response
+        scrape_recent
+        scrape_progress
+        scrape_showcase
+        output
+      end
+
+      def get_response
+        @response = Nokogiri::HTML(open(profile_url+"achievements/")) 
+      end
+
+      def scrape_recent
+        @recent = []
+        6.times do |num|
+          achievement = {}
+          div = response.css("#achv-recent-#{num}")
+          if div
+            achievement[:title] = div.css("div > div").inner_text.strip
+            achievement[:description] = div.inner_text.gsub(achievement[:title], '').strip
+            achievement[:earned] = response.css("#recent-achievements span")[(num*3)+1].inner_text
+
+            @recent << achievement
+          end
+        end
+        @recent
+      end
+
+      def scrape_progress
+        progress_ach = response.css("#progress-module .achievements-progress:nth(2) span")
+        @progress = {
+          liberty_campaign: progress_ach[0].inner_text,
+          exploration:      progress_ach[1].inner_text,
+          custom_game:      progress_ach[2].inner_text,
+          cooperative:      progress_ach[3].inner_text,
+          quick_match:      progress_ach[4].inner_text,
+        }
+      end
+
+      def scrape_showcase
+        @showcase = response.css("#showcase-module .progress-tile").map do |achievement|
+          hsh = { title: achievement.css('.tooltip-title').inner_text.strip }
+          hsh[:description] = achievement.css('div').inner_text.gsub(hsh[:title], '').strip
+          hsh
+        end
+        @showcase
+      end
+
+      def output
+        {
+          recent: @recent,
+          progress: @progress,
+          showcase: @showcase
+        }
+      end
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/base_scraper.rb

@@ -0,0 +1,53 @@
+module BnetScraper
+  module Starcraft2
+    class BaseScraper
+      attr_reader :bnet_id, :account, :region, :bnet_index, :url
+
+      def initialize options = {}
+        if options[:url]
+          extracted_data = options[:url].match(/http:\/\/(.+)\/sc2\/(.+)\/profile\/(.+)\/(\d{1})\/(.[^\/]+)\//)
+          @region     = REGIONS.key({ domain: extracted_data[1], dir: extracted_data[2] })
+          @bnet_id    = extracted_data[3]
+          @bnet_index = extracted_data[4]
+          @account    = extracted_data[5]
+          @url        = options[:url]
+        elsif options[:bnet_id] && options[:account]
+          @bnet_id  = options[:bnet_id]
+          @account  = options[:account]
+          @region   = options[:region] || 'na'
+          if options[:bnet_index]
+            @bnet_index = options[:bnet_index]
+          else
+            set_bnet_index
+          end
+        end
+      end
+
+      # set_bnet_index
+      # 
+      # Because profile URLs have to have a specific bnet_index that is seemingly incalculable,
+      # we must ping both variants to determine the correct bnet_index. We then store that value.
+      def set_bnet_index
+        [1,2].each do |idx|
+          res = Net::HTTP.get_response URI profile_url idx 
+          if res.is_a? Net::HTTPSuccess
+            @bnet_index = idx
+            return
+          end
+        end
+      end
+
+      def profile_url bnet_index = @bnet_index
+        "http://#{region_info[:domain]}/sc2/#{region_info[:dir]}/profile/#{bnet_id}/#{bnet_index}/#{account}/"
+      end
+
+      def region_info
+        REGIONS[region] 
+      end
+
+      def scrape
+        raise NotImplementedError, "Abstract method #scrape called."
+      end
+    end
+  end
+end

data/lib/bnet_scraper/starcraft2/league_scraper.rb

@@ -0,0 +1,50 @@
+module BnetScraper
+  module Starcraft2
+    # LeagueScraper
+    #
+    # Extracts information from an SC2 League URL and scrapes for information.
+    # Example:
+    #   league_data = BnetScraper::Starcraft2::LeagueScraper.new("http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96905")
+    #   league_data # => { bnet_id: '2377239', account: 'Demon', season: '6', size: '4v4', name: "Aleksander Pepper", division: "Diamond", random: false } 
+    #
+    # @param [String] url - The league URL on battle.net
+    # @return [Hash] league_data - Hash of data extracted
+    class LeagueScraper < BaseScraper
+      attr_reader :league_id, :season, :size, :random, :name, :division
+
+      def initialize options = {}
+        super(options)
+
+        if options[:url]
+          @league_id = options[:url].match(/http:\/\/.+\/sc2\/.+\/profile\/.+\/\d{1}\/.+\/ladder\/(.+)(#current-rank)?/).to_a[1]
+        else
+          @league_id = options[:league_id]
+        end
+      end
+
+      def scrape
+        @response = Nokogiri::HTML(open(@url))
+        value = @response.css(".data-title .data-label h3").inner_text().strip 
+        header_regex = /Season (\d{1}) - \s+(\dv\d)( Random)? (\w+)\s+Division (.+)/
+        header_values = value.match(header_regex).to_a
+        header_values.shift()
+        @season, @size, @random, @division, @name = header_values
+        
+        @random = !@random.nil?
+        output
+      end
+
+      def output
+        {
+          season: @season,
+          size: @size,
+          name: @name,
+          division: @division,
+          random: @random,
+          bnet_id: @bnet_id,
+          account: @account
+        }
+      end
+    end
+  end
+end