bnet_scraper 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,65 @@
1
+ module BnetScraper
2
+ module Starcraft2
3
+ ## BnetScraper::Starcraft2::MatchHistoryScraper
4
+ #
5
+ # This pulls the 25 most recent matches played for an account. Note that this is only as up-to-date as battle.net is, and
6
+ # will likely not be as fast as in-game.
7
+ #
8
+ # scraper = BnetScraper::Starcraft2::MatchHistoryScraper.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/')
9
+ # scraper.scrape
10
+ # # => {
11
+ # wins: '15',
12
+ # losses: '10',
13
+ # matches: [
14
+ # { map_name: 'Bx Monobattle - Sand Canyon (Fix)', outcome: :win, type: 'Custom', date: '3/12/2012' },
15
+ # { map_name: 'Deadlock Ridge', outcome: :loss, type: '4v4', date: '3/12/2012' },
16
+ # { map_name: 'District 10', outcome: :win, type: '4v4', date: '3/12/2012' },
17
+ # # ...
18
+ # ]
19
+ # }
20
+ class MatchHistoryScraper < BaseScraper
21
+ attr_reader :matches, :wins, :losses, :response
22
+
23
+ def match_url
24
+ profile_url + "matches"
25
+ end
26
+
27
+ def get_response
28
+ @response = Nokogiri::HTML(open(match_url))
29
+ end
30
+
31
+ def scrape
32
+ get_response
33
+ @matches = []
34
+ @wins = 0
35
+ @losses = 0
36
+
37
+ response.css('.match-row').each do |m|
38
+ match = {}
39
+
40
+ cells = m.css('td')
41
+ match[:map_name] = cells[1].inner_text
42
+ match[:type] = cells[2].inner_text
43
+ match[:outcome] = (cells[3].inner_text.strip == 'Win' ? :win : :loss)
44
+ match[:date] = cells[4].inner_text.strip
45
+ @matches << match
46
+ if match[:outcome] == :win
47
+ @wins += 1
48
+ else
49
+ @losses += 1
50
+ end
51
+ output
52
+ end
53
+
54
+ end
55
+
56
+ def output
57
+ {
58
+ matches: @matches,
59
+ wins: @wins,
60
+ losses: @losses
61
+ }
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,63 @@
1
+ module BnetScraper
2
+ module Starcraft2
3
+ # ProfileScraper
4
+ #
5
+ # Scrapes SC2 profile data from battle.net and returns it as a hash. Example:
6
+ # profile_data = BnetScraper::Starcraft2::ProfileScraper.new('2377239', 'Demon')
7
+ # profile_data # => { bnet_id: '2377239', account: 'Demon', race: 'Protoss',
8
+ # wins: '684', achievements: '3260', leagues: [], bnet_index: 1 }
9
+ #
10
+ # One thing of note is the bnet_index. In Battle.net URLs, there is a single-digit index
11
+ # used on accounts (1 or 2). This index is seemingly arbitrary, but critical to properly
12
+ # accessing the data.
13
+ #
14
+ # ProfileScraper requires that either you pass the URL of the profile, or the bnet_id and
15
+ # account name of the profile. The URL scheme is as such:
16
+ #
17
+ # http://<REGION_DOMAIN>/sc2/<REGION_LANG>/profile/<BNET_ID>/<BNET_INDEX>/<ACCOUNT>/
18
+ #
19
+ # Using this URL we can extract the critical information. However, sometimes we do not have
20
+ # the URL and have to make do with a bnet_id and account. This is the bare minimum needed,
21
+ # unless the account is in a region other than 'na'. In such cases, region all needs to be passed.
22
+ class ProfileScraper < BaseScraper
23
+ attr_reader :achievement_points, :wins, :race, :leagues
24
+ def scrape
25
+ get_profile_data
26
+ get_league_list
27
+ output
28
+ end
29
+
30
+ def get_profile_data
31
+ response = Nokogiri::HTML(open(profile_url))
32
+
33
+ @race = response.css("#season-snapshot .module-footer a").first().inner_html()
34
+ @wins = response.css("#career-stats h2").inner_html()
35
+ @achievement_points = response.css("#profile-header h3").inner_html()
36
+ end
37
+
38
+ def get_league_list
39
+ response = Nokogiri::HTML(open(profile_url + "ladder/leagues"))
40
+
41
+ @leagues = response.css("a[href*='#current-rank']").map do |league|
42
+ {
43
+ name: league.inner_html().strip,
44
+ id: league.attr('href').sub('#current-rank',''),
45
+ href: "#{profile_url}ladder/#{league.attr('href')}"
46
+ }
47
+ end
48
+ end
49
+
50
+ def output
51
+ {
52
+ bnet_id: @bnet_id,
53
+ account: @account,
54
+ bnet_index: @bnet_index,
55
+ race: @race,
56
+ wins: @wins,
57
+ achievement_points: @achievement_points,
58
+ leagues: @leagues
59
+ }
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,5 @@
1
+ $:.push File.join(File.dirname(__FILE__), '..', 'lib')
2
+
3
+ require 'bnet_scraper'
4
+ Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each {|f| require f }
5
+
@@ -0,0 +1,99 @@
1
+ require 'spec_helper'
2
+
3
+ describe BnetScraper::Starcraft2::AchievementScraper do
4
+ let(:url) { 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/achievements/' }
5
+ subject { BnetScraper::Starcraft2::AchievementScraper.new(url: url) }
6
+
7
+ it_behaves_like 'an SC2 Scraper' do
8
+ let(:scraper_class) { BnetScraper::Starcraft2::AchievementScraper }
9
+ let(:subject) { scraper_class.new(url: url) }
10
+ end
11
+
12
+ describe '#get_response' do
13
+ it 'should get the HTML response to be scraped' do
14
+ subject.response.should be_nil
15
+ subject.get_response
16
+ subject.response.should_not be_nil
17
+ end
18
+ end
19
+
20
+ describe '#scrape' do
21
+ it 'should call get_response and trigger scraper methods' do
22
+ subject.should_receive(:get_response)
23
+ subject.should_receive(:scrape_progress)
24
+ subject.should_receive(:scrape_recent)
25
+ subject.should_receive(:scrape_showcase)
26
+ subject.scrape
27
+ end
28
+ end
29
+
30
+ describe '#scrape_showcase' do
31
+ before :each do
32
+ subject.get_response
33
+ subject.scrape_showcase
34
+ end
35
+
36
+ it 'should set the showcase' do
37
+ subject.showcase.should have(5).achievements
38
+ end
39
+ end
40
+
41
+ describe '#scrape_recent' do
42
+ before :each do
43
+ subject.get_response
44
+ subject.scrape_recent
45
+ end
46
+
47
+ it 'should have the title of the achievement' do
48
+ subject.recent[0][:title].should == 'Blink of an Eye'
49
+ end
50
+
51
+ it 'should have the description of the achievement' do
52
+ # this is a cop-out because the string contains UTF-8. Please fix this. - Cad
53
+ subject.recent[0][:description].should be_a String
54
+ end
55
+
56
+ it 'should have the date the achievement was earned' do
57
+ subject.recent[0][:earned].should == '3/5/2012'
58
+ end
59
+ end
60
+
61
+ describe '#scrape_progress' do
62
+ before :each do
63
+ subject.get_response
64
+ subject.scrape_progress
65
+ end
66
+
67
+ it 'should set the liberty campaign progress' do
68
+ subject.progress[:liberty_campaign].should == '1580'
69
+ end
70
+
71
+ it 'should set the exploration progress' do
72
+ subject.progress[:exploration].should == '480'
73
+ end
74
+
75
+ it 'should set the custom game progress' do
76
+ subject.progress[:custom_game].should == '330'
77
+ end
78
+
79
+ it 'should set the cooperative progress' do
80
+ subject.progress[:cooperative].should == '660'
81
+ end
82
+
83
+ it 'should set the quick match progress' do
84
+ subject.progress[:quick_match].should == '170'
85
+ end
86
+ end
87
+
88
+ describe '#output' do
89
+ it 'should return the scraped data when scrape has been called' do
90
+ subject.scrape
91
+ expected = {
92
+ recent: subject.recent,
93
+ showcase: subject.showcase,
94
+ progress: subject.progress
95
+ }
96
+ subject.output.should == expected
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,14 @@
1
+ require 'spec_helper'
2
+
3
+ describe BnetScraper::Starcraft2::BaseScraper do
4
+ it_behaves_like 'an SC2 Scraper' do
5
+ let(:scraper_class) { BnetScraper::Starcraft2::BaseScraper }
6
+ let(:subject) { scraper_class.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/') }
7
+ end
8
+
9
+ describe '#scrape' do
10
+ it 'should raise an error calling scrape' do
11
+ expect { subject.scrape }.to raise_error NotImplementedError
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,71 @@
1
+ require 'spec_helper'
2
+
3
+ describe BnetScraper::Starcraft2::LeagueScraper do
4
+ let(:url) { "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/12345" }
5
+ subject { BnetScraper::Starcraft2::LeagueScraper.new(url: url) }
6
+
7
+ it_behaves_like 'an SC2 Scraper' do
8
+ let(:scraper_class) { BnetScraper::Starcraft2::LeagueScraper }
9
+ let(:subject) { scraper_class.new(url: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/12345") }
10
+ end
11
+
12
+ describe '#initialize' do
13
+ it 'should dissect the league_id from the URL' do
14
+ subject.league_id.should == '12345'
15
+ end
16
+ end
17
+
18
+ describe '#scrape' do
19
+ it 'should set the season value' do
20
+ subject.season.should be_nil
21
+ subject.scrape
22
+ subject.season.should == '6'
23
+ end
24
+
25
+ it 'should set the name' do
26
+ subject.name.should be_nil
27
+ subject.scrape
28
+ subject.name.should == 'Aleksander Pepper'
29
+ end
30
+
31
+ it 'should set the divison' do
32
+ subject.division.should be_nil
33
+ subject.scrape
34
+ subject.division.should == 'Diamond'
35
+ end
36
+
37
+ it 'should set the size' do
38
+ subject.size.should be_nil
39
+ subject.scrape
40
+ subject.size.should == '4v4'
41
+ end
42
+
43
+ it 'should set if player is random' do
44
+ subject.random.should be_nil
45
+ subject.scrape
46
+ subject.random.should be_false
47
+ end
48
+
49
+ it 'should call output' do
50
+ subject.should_receive(:output)
51
+ subject.scrape
52
+ end
53
+ end
54
+
55
+ describe '#output' do
56
+ it 'should return a hash of league data' do
57
+ expected = {
58
+ season: '6',
59
+ name: 'Aleksander Pepper',
60
+ division: 'Diamond',
61
+ size: '4v4',
62
+ random: false,
63
+ bnet_id: '2377239',
64
+ account: 'Demon'
65
+ }
66
+
67
+ subject.scrape
68
+ subject.output.should == expected
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,53 @@
1
+ require 'spec_helper'
2
+
3
+ describe BnetScraper::Starcraft2::MatchHistoryScraper do
4
+ let(:url) { 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/matches' }
5
+ subject { BnetScraper::Starcraft2::MatchHistoryScraper.new(url: url) }
6
+
7
+ it_behaves_like 'an SC2 Scraper' do
8
+ let(:scraper_class) { BnetScraper::Starcraft2::MatchHistoryScraper }
9
+ let(:scraper) { scraper_class.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/matches') }
10
+ end
11
+
12
+ describe '#scrape' do
13
+
14
+ before :each do
15
+ subject.scrape
16
+ end
17
+
18
+ it 'sets the matches to an array of match data' do
19
+ subject.matches.should have(25).matches
20
+ end
21
+
22
+ it 'should set the name of the map for each match' do
23
+ subject.matches[0][:map_name].should == 'Bx Monobattle - Sand Canyon (Fix)'
24
+ end
25
+
26
+ it 'should set the type of the match for each match' do
27
+ subject.matches[0][:type].should == 'Custom'
28
+ end
29
+
30
+ it 'should set the outcome of each match' do
31
+ subject.matches[0][:outcome].should == :win
32
+ end
33
+
34
+ it 'should set the match date of each match' do
35
+ subject.matches[0][:date].should == '3/12/2012'
36
+ end
37
+ end
38
+
39
+ describe '#output' do
40
+ before :each do
41
+ subject.scrape
42
+ end
43
+
44
+ it 'should return a hash of the scraped match data' do
45
+ expected = {
46
+ matches: subject.matches,
47
+ wins: subject.wins,
48
+ losses: subject.losses
49
+ }
50
+ subject.output.should == expected
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,128 @@
1
+ require 'spec_helper'
2
+
3
+ describe BnetScraper::Starcraft2::ProfileScraper do
4
+ it_behaves_like 'an SC2 Scraper' do
5
+ let(:scraper_class) { BnetScraper::Starcraft2::BaseScraper }
6
+ let(:subject) { scraper_class.new(url: 'http://us.battle.net/sc2/en/profile/2377239/1/Demon/') }
7
+ end
8
+
9
+ subject { BnetScraper::Starcraft2::ProfileScraper.new(bnet_id: '2377239', account: 'Demon') }
10
+
11
+ describe '#get_profile_data' do
12
+ it 'should set the race, wins, and achievements attributes' do
13
+ subject.instance_variable_get(:@race).should be_nil
14
+ subject.instance_variable_get(:@achievement_points).should be_nil
15
+ subject.instance_variable_get(:@wins).should be_nil
16
+
17
+ subject.get_profile_data
18
+
19
+ subject.instance_variable_get(:@race).should == 'Protoss'
20
+ subject.instance_variable_get(:@achievement_points).should == '3630'
21
+ subject.instance_variable_get(:@wins).should == '684'
22
+ end
23
+ end
24
+
25
+ describe 'get_league_list' do
26
+ it 'should set an array of leagues' do
27
+ subject.instance_variable_get(:@leagues).should be_nil
28
+ subject.get_league_list
29
+
30
+ subject.instance_variable_get(:@leagues).should have(12).leagues
31
+ end
32
+ end
33
+
34
+ describe '#scrape' do
35
+ it 'should call get_profile_data' do
36
+ subject.should_receive(:get_profile_data)
37
+ subject.scrape
38
+ end
39
+ it 'should call get_league_list' do
40
+ subject.should_receive(:get_league_list)
41
+ subject.scrape
42
+ end
43
+
44
+ it 'should call output' do
45
+ subject.should_receive(:output)
46
+ subject.scrape
47
+ end
48
+ end
49
+
50
+ describe '#output' do
51
+ it 'should extract profile data from the response' do
52
+ expected = {
53
+ bnet_id: '2377239',
54
+ account: 'Demon',
55
+ bnet_index: 1,
56
+ race: 'Protoss',
57
+ wins: '684',
58
+ achievement_points: '3630',
59
+ leagues: [
60
+ {
61
+ name: "1v1 Platinum Rank 95",
62
+ id: "96905",
63
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96905#current-rank"
64
+ },
65
+ {
66
+ name: "2v2 Random Platinum ...",
67
+ id: "96716",
68
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96716#current-rank"
69
+ },
70
+ {
71
+ name: "2v2 Diamond Rank 45",
72
+ id: "98162",
73
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/98162#current-rank"
74
+ },
75
+ {
76
+ name: "2v2 Silver Rank 8",
77
+ id: "97369",
78
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/97369#current-rank"
79
+ },
80
+ {
81
+ name: "3v3 Random Gold Rank...",
82
+ id: "96828",
83
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96828#current-rank"
84
+ },
85
+ {
86
+ name: "3v3 Diamond Rank 56",
87
+ id: "97985",
88
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/97985#current-rank"
89
+ },
90
+ {
91
+ name: "3v3 Silver Rank 5",
92
+ id: "98523",
93
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/98523#current-rank"
94
+ },
95
+ {
96
+ name: "3v3 Platinum Rank 88",
97
+ id: "96863",
98
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96863#current-rank"
99
+ },
100
+ {
101
+ name: "3v3 Gold Rank 75",
102
+ id: "97250",
103
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/97250#current-rank"
104
+ },
105
+ {
106
+ name: "4v4 Random Platinum ...",
107
+ id: "96830",
108
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/96830#current-rank"
109
+ },
110
+ {
111
+ name: "4v4 Gold Rank 38",
112
+ id: "98336",
113
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/98336#current-rank"
114
+ },
115
+ {
116
+ name: "4v4 Diamond Rank 54",
117
+ id: "98936",
118
+ href: "http://us.battle.net/sc2/en/profile/2377239/1/Demon/ladder/98936#current-rank"
119
+ }
120
+ ]
121
+ }
122
+
123
+ subject.output.should == { bnet_id: '2377239', account: 'Demon', bnet_index: 1, race: nil, wins: nil, achievement_points: nil, leagues: nil }
124
+ subject.scrape
125
+ subject.output.should == expected
126
+ end
127
+ end
128
+ end