trophy-scraper 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
@@ -0,0 +1,6 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
6
+ proj/
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License
2
+
3
+ Copyright (c) 2009 Vince Cima
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,39 @@
1
+ ## TrophyScraper
2
+ ###Collect and store trophy data for Playstation Network users.
3
+
4
+ ## Dependencies
5
+ * Sqlite
6
+ * DataMapper
7
+ * Mechanize
8
+
9
+ ## Installation
10
+ ### Generic
11
+ gem install trophy-scraper
12
+ ### Ubuntu/Debian
13
+ sudo aptitude install ruby ruby-dev rubygems sqlite3 libsqlite3-dev make libopenssl-ruby libxslt-dev libxml2-dev
14
+ sudo gem install trophy-scraper
15
+ The rubygems wrapped exectuable will be located at:
16
+
17
+ /var/lib/gems/1.8/bin/trophy-scraper
18
+ Make trophy-scraper available in the default PATH:
19
+
20
+ sudo ln -s /var/lib/gems/1.8/bin/trophy-scraper /usr/bin/trophy-scraper
21
+
22
+ ## Usage
23
+ ### Adding a user / Update a single user
24
+ trophy-scraper --username "Insert Username Here"
25
+ ### Updating all users
26
+ trophy-scraper
27
+ ### Specifying a custom database location
28
+ trophy-scraper --db "sqlite3://home/joe/trophy_scraper.db"
29
+ ### Changing the logging options
30
+ trophy-scraper --log_level debug --log_dest "~/trophy-scraper.log" --db_log_level error --db_log_dest "~/trophy-scraper-db.log"
31
+ ### Help
32
+ trophy-scraper --help
33
+
34
+ ## Limitations
35
+ 1. Some trophies are marked as "spoilers" and will never show their name and description.
36
+ 1. Some games simply don't show up in the list of games played. Their trophies will contribute to a player's total however.
37
+
38
+ ## Thanks
39
+ Thanks to the Haml and Jekyll projects for a little bit of code and lot of great examples on how to layout a Ruby project.
@@ -0,0 +1,64 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "trophy-scraper"
8
+ gem.summary = %Q{Collect and store trophy data for Playstation Network users.}
9
+ gem.description = %Q{Collect and store trophy data for Playstation Network users.}
10
+ gem.email = "contact@vincecima.com"
11
+ gem.homepage = "http://vincecima.com/blog/trophy-scraper/"
12
+ gem.authors = ["Vince Cima"]
13
+ gem.rubyforge_project = "trophy-scraper"
14
+ gem.add_dependency("dm-core",">= 0.9.11")
15
+ gem.add_dependency("do_sqlite3",">= 0.9.12")
16
+ gem.add_dependency("mechanize",">= 0.9.3")
17
+ gem.rubyforge_project = "trophy-scraper"
18
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
19
+ end
20
+ Jeweler::GemcutterTasks.new
21
+ Jeweler::RubyforgeTasks.new do |rubyforge|
22
+ rubyforge.doc_task = "rdoc"
23
+ end
24
+ rescue LoadError
25
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
26
+ end
27
+
28
+ require 'rake/testtask'
29
+ Rake::TestTask.new(:test) do |test|
30
+ test.libs << 'lib' << 'test'
31
+ test.pattern = 'test/**/*_test.rb'
32
+ test.verbose = true
33
+ end
34
+
35
+ begin
36
+ require 'rcov/rcovtask'
37
+ Rcov::RcovTask.new do |test|
38
+ test.libs << 'test'
39
+ test.pattern = 'test/**/*_test.rb'
40
+ test.verbose = true
41
+ end
42
+ rescue LoadError
43
+ task :rcov do
44
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
45
+ end
46
+ end
47
+
48
+ task :test => :check_dependencies
49
+
50
+ task :default => :test
51
+
52
+ require 'rake/rdoctask'
53
+ Rake::RDocTask.new do |rdoc|
54
+ if File.exist?('VERSION')
55
+ version = File.read('VERSION')
56
+ else
57
+ version = ""
58
+ end
59
+
60
+ rdoc.rdoc_dir = 'rdoc'
61
+ rdoc.title = "trophy-scraper #{version}"
62
+ rdoc.rdoc_files.include('README*')
63
+ rdoc.rdoc_files.include('lib/**/*.rb')
64
+ end
data/SQL ADDED
@@ -0,0 +1,6 @@
1
+ SELECT g.title, COUNT(g.title)
2
+ FROM unlocks u, trophies t, games g
3
+ WHERE user_username = 'username'
4
+ AND u.trophy_id = t.id
5
+ AND t.game_id = g.id
6
+ GROUP BY g.title
data/TODO.md ADDED
@@ -0,0 +1,10 @@
1
+ ## User Management
2
+ 1. Add the ability to remove a user from the data store.
3
+ 1. Add/Remove groups of users without triggering an update.
4
+
5
+ ## Trophy Handling
6
+ 1. Detect trophies that are displayed on Playstation.com and log out message.
7
+ 1. Better handle "spoiler" trophies.
8
+
9
+ ## Scraper
10
+ 1. Add detection of page structure changes.
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.0.0
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
4
+
5
+ help = <<HELP
6
+ TrophyScraper downloads trophy information for Playstation Network Accounts.
7
+
8
+ Basic Command Line Usage:
9
+ trophy-scraper # Update all users in database.
10
+ trophy-scraper --username "username" # Update "username".
11
+ trophy-scraper --db "sqlite3::memory:" # DataMapper db string.
12
+
13
+ Configuration is read from '~/.trophy_scraper.yml' but can be overriden
14
+ using the following options:
15
+
16
+ HELP
17
+
18
+ require 'optparse'
19
+ require 'trophy_scraper'
20
+
21
+ options = {}
22
+ opts = OptionParser.new do |opts|
23
+ opts.banner = help
24
+
25
+ opts.on("--config [PATH]", "Full path to yml configuration file.") do |path|
26
+ options['config'] = path
27
+ end
28
+
29
+ opts.on("--db [DB_STRING]", "DataMapper database string.") do |db_string|
30
+ options['db'] = db_string
31
+ end
32
+
33
+ opts.on("--username [USERNAME]", "Username to run the trophy update for.") do |username|
34
+ options['username'] = username
35
+ end
36
+
37
+ opts.on("--log-level [LOG_LEVEL]", "Standard Ruby logger output level.") do |log_level|
38
+ if log_level.downcase == "debug"
39
+ options['log_level'] = Logger::DEBUG
40
+ elsif log_level.downcase == "info"
41
+ options['log_level'] = Logger::INFO
42
+ elsif log_level.downcase == "warn"
43
+ options['log_level'] = Logger::WARN
44
+ elsif log_level.downcase == "error"
45
+ options['log_level'] = Logger::ERROR
46
+ elsif log_level.downcase == "fatal"
47
+ options['log_level'] = Logger::FATAL
48
+ end
49
+ end
50
+
51
+ opts.on("--log-dest [LOG_DEST]", "STDOUT, STDERR or full path to log file.") do |log_dest|
52
+ if log_dest.downcase == "STDOUT".downcase
53
+ options['log_dest'] = STDOUT
54
+ elsif log_dest.downcase == "STDERR".downcase
55
+ options['log_dest'] = STDERR
56
+ else
57
+ options['log_dest'] = File.expand_path(log_dest)
58
+ end
59
+ end
60
+
61
+ opts.on("--db-log-level [LOG_LEVEL]", "Ruby logger output level for database.") do |log_level|
62
+ if log_level.downcase == "off"
63
+ options['db_log_level'] = :off
64
+ elsif log_level.downcase == "debug"
65
+ options['db_log_level'] = :debug
66
+ elsif log_level.downcase == "info"
67
+ options['db_log_level'] = :info
68
+ elsif log_level.downcase == "warn"
69
+ options['db_log_level'] = :warn
70
+ elsif log_level.downcase == "error"
71
+ options['db_log_level'] = :error
72
+ elsif log_level.downcase == "fatal"
73
+ options['db_log_level'] = :fatal
74
+ end
75
+ end
76
+
77
+ opts.on("--db-log-dest [LOG_DEST]", "STDOUT, STDERR or full path to db log file.") do |db_log_dest|
78
+ if db_log_dest.downcase == "STDOUT".downcase
79
+ options['db_log_dest'] = STDOUT
80
+ elsif db_log_dest.downcase == "STDERR".downcase
81
+ options['db_log_dest'] = STDERR
82
+ else
83
+ options['db_log_dest'] = File.expand_path(db_log_dest)
84
+ end
85
+ end
86
+ end
87
+
88
+ # Read command line options into `options` hash
89
+ opts.parse!
90
+
91
+ options = TrophyScraper.configuration(options)
92
+
93
+ controller = Controller.new(options)
94
+ controller.process
@@ -0,0 +1,68 @@
1
+ $:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
2
+
3
+ # core
4
+
5
+ # stdlib
6
+ require "logger"
7
+
8
+ # 3rd party
9
+ require 'dm-core'
10
+ require 'mechanize'
11
+
12
+ # internal requires
13
+ require "trophy_scraper/controller"
14
+ require "trophy_scraper/core_ext"
15
+ require "trophy_scraper/data_store"
16
+ require "trophy_scraper/scraper"
17
+ require "trophy_scraper/models/meta"
18
+ require "trophy_scraper/models/user"
19
+ require "trophy_scraper/models/user_stat"
20
+ require "trophy_scraper/models/unlock"
21
+ require "trophy_scraper/models/trophy"
22
+ require "trophy_scraper/models/game"
23
+ require "trophy_scraper/models/trophy_level"
24
+
25
+ module TrophyScraper
26
+ # Default options. Overriden by values in _config.yml or command-line opts.
27
+ # (Strings rather symbols used for compatability with YAML)
28
+ DEFAULTS = {
29
+ 'config' => '~/.trophy_scraper.yml',
30
+ 'db' => '',
31
+ 'log_level' => Logger::INFO,
32
+ 'log_dest' => STDOUT,
33
+ 'db' => "sqlite3://%s/trophy_scraper.db" % File.expand_path("~"),
34
+ 'db_log_level' => "fatal",
35
+ 'db_log_dest' => STDOUT
36
+ }
37
+
38
+ # Generate a TrophyScraper configuration Hash by merging the default options
39
+ # with anything in config.yml, and adding the given options on top
40
+ # +override+ is a Hash of config directives
41
+ #
42
+ # Returns Hash
43
+ def self.configuration(override)
44
+ # config.yml may override default source location, but until
45
+ # then, we need to know where to look for config.yml
46
+ source = override['config'] || TrophyScraper::DEFAULTS['config']
47
+
48
+ # Get configuration from <source>
49
+ config_file = File.expand_path(source)
50
+ begin
51
+ config = YAML.load_file(config_file)
52
+ raise "Invalid configuration - #{config_file}" if !config.is_a?(Hash)
53
+ STDOUT.puts "Configuration from #{config_file}"
54
+ rescue => err
55
+ STDERR.puts "WARNING: Could not read configuration. Using defaults (and options)."
56
+ STDERR.puts "\t" + err.to_s
57
+ config = {}
58
+ end
59
+
60
+ # Merge DEFAULTS < config.yml < override
61
+ TrophyScraper::DEFAULTS.deep_merge(config).deep_merge(override)
62
+ end
63
+
64
+ def self.version
65
+ yml = YAML.load(File.read(File.join(File.dirname(__FILE__), *%w[.. VERSION.yml])))
66
+ "#{yml[:major]}.#{yml[:minor]}.#{yml[:patch]}"
67
+ end
68
+ end
@@ -0,0 +1,91 @@
1
+ class Controller
2
+ def initialize(options)
3
+ @options = options.clone
4
+ $logger = Logger.new(@options['log_dest'])
5
+ $logger.level = @options['log_level']
6
+ end
7
+
8
+ def process
9
+ #Initialize connection to db and do any version migration needed.
10
+ data_store = DataStore.new(@options['db'], @options['db_log_level'], @options['db_log_dest'])
11
+
12
+ users_to_check = get_users_to_check
13
+ trophy_levels = TrophyLevel.all
14
+
15
+ users_to_check.each do |user|
16
+ $logger.info("Preparing to update #{user.username}.")
17
+ scraper = Scraper.new(user.username)
18
+
19
+ user_summary_stats = scraper.get_user_summary_stats
20
+ if user_summary_stats.total_trophies > user.unlocks.size
21
+ $logger.info("Website reports more trophies for #{user.username} than data store.")
22
+ user.user_stats << user_summary_stats
23
+
24
+ games_summary_stats = scraper.get_game_summary_stats()
25
+ games_summary_stats.each_pair do |found_game, trophies_earned|
26
+ game = Game.first(:title => found_game.title)
27
+
28
+ if game == nil
29
+ game = found_game
30
+
31
+ game.save
32
+ $logger.debug("Saved new game: #{game.title}")
33
+ end
34
+
35
+ trophy_info = scraper.get_game_detail_stats(found_game.uri, trophy_levels)
36
+ trophy_info.each_pair do |found_trophy, earned_date|
37
+ trophy = game.trophies.first(:name => found_trophy.name, :game_id => game.id, :trophy_level_id => found_trophy.trophy_level_id)
38
+ if trophy == nil
39
+ trophy = found_trophy
40
+
41
+ game.trophies << trophy
42
+ trophy.save
43
+ $logger.debug("Saved new trophy: #{trophy.name}")
44
+ end
45
+
46
+ if earned_date != nil
47
+ unlock = Unlock.first(:user_username => user.username, :trophy_id => trophy.id)
48
+ if unlock == nil
49
+ unlock = Unlock.new(:earned_on => earned_date)
50
+ trophy.unlocks << unlock
51
+ user.unlocks << unlock
52
+ unlock.save
53
+ $logger.info("#{user.username} has unlocked '#{trophy.name}' in '#{game.title}'.")
54
+ end
55
+ end
56
+ end
57
+ end
58
+ elsif user_summary_stats.total_trophies == user.unlocks.size
59
+ $logger.info("Same number of trophies on website and in data store. Nothing to update for #{user.username}")
60
+ else
61
+ $logger.warn("Website reports less trophies than data store for #{user.username}. Something is wrong.")
62
+ end
63
+
64
+ user.last_updated = Time.now
65
+ user.save
66
+ end
67
+
68
+ $logger.close
69
+ end
70
+
71
+ private
72
+
73
+ def get_users_to_check
74
+ users_to_check = Array.new
75
+ if @options['username'] != nil
76
+ user = User.get(@options['username'])
77
+ if user == nil
78
+ user = User.new(:username => @options['username'])
79
+ user.save
80
+ $logger.info("#{user.username} is a new user.")
81
+ end
82
+ users_to_check.push(user)
83
+ $logger.info("#{user.username} will be the only user checked.")
84
+ else
85
+ users_to_check = User.all
86
+ $logger.info("All users in data store will be checked.")
87
+ end
88
+
89
+ return users_to_check
90
+ end
91
+ end
@@ -0,0 +1,22 @@
1
+ class Hash
2
+ # Merges self with another hash, recursively.
3
+ #
4
+ # This code was lovingly stolen from some random gem:
5
+ # http://gemjack.com/gems/tartan-0.1.1/classes/Hash.html
6
+ #
7
+ # Thanks to whoever made it.
8
+ def deep_merge(hash)
9
+ target = dup
10
+
11
+ hash.keys.each do |key|
12
+ if hash[key].is_a? Hash and self[key].is_a? Hash
13
+ target[key] = target[key].deep_merge(hash[key])
14
+ next
15
+ end
16
+
17
+ target[key] = hash[key]
18
+ end
19
+
20
+ target
21
+ end
22
+ end
@@ -0,0 +1,40 @@
1
+ class DataStore
2
+ @@DB_VERSION = 1
3
+ def initialize(connection_string, logger_level, logger_dest)
4
+ DataMapper::Logger.new(logger_dest, logger_level)
5
+ DataMapper.setup(:default, connection_string)
6
+
7
+ begin
8
+ version_meta = Meta.get("version")
9
+ rescue
10
+ #TODO, only eat the table not found error
11
+ end
12
+ if version_meta == nil
13
+ $logger.info("Database is empty, creating schema.")
14
+ DataMapper.auto_migrate!
15
+ version_meta = Meta.new(:name => "version", :value => @@DB_VERSION)
16
+ version_meta.save
17
+ insert_reference_data
18
+ elsif version_meta.value.to_i < @@DB_VERSION
19
+ $logger.info("Database schema version is older than code's, gracefully updating.")
20
+ DataMapper.auto_upgrade!
21
+ version_meta.value = @@DB_VERSION
22
+ version_meta.save
23
+ elsif version_meta.value.to_i > @@DB_VERSION
24
+ #TODO Code is older than db, warn and abort.
25
+ $logger.warn("Database schema version is newer than code's, aborting.")
26
+ else
27
+ $logger.info("Database schema is up to date.")
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def insert_reference_data
34
+ TrophyLevel.new(:name => "Platinum").save
35
+ TrophyLevel.new(:name => "Gold").save
36
+ TrophyLevel.new(:name => "Silver").save
37
+ TrophyLevel.new(:name => "Bronze").save
38
+ TrophyLevel.new(:name => "Hidden").save
39
+ end
40
+ end
@@ -0,0 +1,11 @@
1
+ class Game
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :title, String, :nullable => false
6
+ property :logo, Object
7
+
8
+ has n, :trophies
9
+
10
+ attr_accessor :uri, :logo_uri
11
+ end
@@ -0,0 +1,6 @@
1
+ class Meta
2
+ include DataMapper::Resource
3
+
4
+ property :name, String, :key => true
5
+ property :value, String
6
+ end
@@ -0,0 +1,19 @@
1
+ class Trophy
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :name, String, :nullable => false
6
+ property :description, String, :nullable => false
7
+ property :icon, Object
8
+
9
+ has n, :unlocks
10
+ has n, :users, :through => :unlocks
11
+ belongs_to :game
12
+ belongs_to :trophy_level
13
+
14
+ attr_accessor :icon_uri
15
+
16
+ def to_s
17
+ return "id = #{@id}, name = #{@name}, description = #{@description}"
18
+ end
19
+ end
@@ -0,0 +1,7 @@
1
+ class TrophyLevel
2
+ include DataMapper::Resource
3
+ property :id, Serial
4
+ property :name, String, :nullable => false
5
+
6
+ has n, :trophies
7
+ end
@@ -0,0 +1,8 @@
1
+ class Unlock
2
+ include DataMapper::Resource
3
+ property :id, Serial
4
+ property :earned_on, DateTime, :nullable => false
5
+
6
+ belongs_to :trophy
7
+ belongs_to :user
8
+ end
@@ -0,0 +1,10 @@
1
+ class User
2
+ include DataMapper::Resource
3
+
4
+ property :username, String, :key => true
5
+ property :last_updated, Time
6
+
7
+ has n, :user_stats, :model => "UserStat"
8
+ has n, :unlocks
9
+ has n, :trophies, :through => :unlocks
10
+ end
@@ -0,0 +1,21 @@
1
+ class UserStat
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :number_of_bronze, Integer
6
+ property :number_of_silver, Integer
7
+ property :number_of_gold, Integer
8
+ property :number_of_platinum, Integer
9
+ property :level, Integer
10
+ property :percent_to_next_level, Integer
11
+
12
+ belongs_to :user
13
+
14
+ def total_trophies
15
+ return @number_of_bronze + @number_of_silver + @number_of_gold + @number_of_platinum
16
+ end
17
+
18
+ def to_s
19
+ return "percent_to_next_level = #{@percent_to_next_level}, level = #{@level}, #bronze = #{@number_of_bronze}, #silver = #{@number_of_silver}, #gold = #{@number_of_gold}, #platinum = #{@number_of_platinum}"
20
+ end
21
+ end
@@ -0,0 +1,95 @@
1
+ class Scraper
2
+ TROPHY_PAGE_URL = "http://profiles.us.playstation.com/playstation/psn/profiles/%s"
3
+ TROPHY_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_trophies_data"
4
+ TROPHY_DETAILED_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_title_details_data"
5
+
6
+ def initialize(username)
7
+ @username = username
8
+ @agent = WWW::Mechanize.new
9
+ @update_date = Time.now
10
+ @account_trophy_page = nil
11
+ self.request_account_trophy_page
12
+ end
13
+
14
+ def get_user_summary_stats
15
+ stats = UserStat.new
16
+ stats.level = @account_trophy_page.search("#leveltext").text.strip
17
+ stats.percent_to_next_level = @account_trophy_page.search(".progresstext").text.strip.delete("%")
18
+ stats.number_of_bronze = @account_trophy_page.search(".podium .bronze").text.strip.split[0]
19
+ stats.number_of_silver = @account_trophy_page.search(".podium .silver").text.strip.split[0]
20
+ stats.number_of_gold = @account_trophy_page.search(".podium .gold").text.strip.split[0]
21
+ stats.number_of_platinum = @account_trophy_page.search(".podium .platinum").text.strip.split[0]
22
+ $logger.debug("Found user stats #{stats}")
23
+ return stats
24
+ end
25
+
26
+ def get_game_summary_stats()
27
+ trophy_data = @agent.get(TROPHY_DATA_URL % [@username])
28
+ summary_stats = Hash.new
29
+
30
+ game_slots = trophy_data.search(".slot")
31
+ game_slots.each { |game_data|
32
+ game = Game.new
33
+ game.title = game_data.search(".titletext a").text.strip
34
+ game.uri = game_data.search(".titletext a").first[:href].strip
35
+ game.logo_uri = game_data.search(".titlelogo img").first[:src].strip
36
+ #TODO Make sure we got the right amount of trophy columns
37
+ trophies_earned = Hash.new
38
+ trophies_earned[:bronze] = game_data.search(".trophycount.normal .trophycontent")[0].text.strip
39
+ trophies_earned[:silver] = game_data.search(".trophycount.normal .trophycontent")[1].text.strip
40
+ trophies_earned[:gold] = game_data.search(".trophycount.normal .trophycontent")[2].text.strip
41
+ trophies_earned[:platinum] = game_data.search(".trophycount.normal .trophycontent")[3].text.strip
42
+
43
+ summary_stats[game] = trophies_earned
44
+ }
45
+ return summary_stats
46
+ end
47
+
48
+ def get_game_detail_stats(game_uri, trophy_levels)
49
+ trophy_info = Hash.new
50
+ hidden_trophy_counter = 1
51
+
52
+ titleId = game_uri.split("/")[-1]
53
+ trophy_data = @agent.post(TROPHY_DETAILED_DATA_URL % [@username], "titleId" => titleId, "sortBy" => "id_asc")
54
+
55
+ trophy_data.search(".slotcontent").each { |slot|
56
+ trophy = Trophy.new
57
+ trophy.name = slot.search(".trophyTitleSortField").text.strip
58
+ trophy.description = slot.search(".subtext").text.strip
59
+
60
+ # Game is a "spoiler trophy", ignore for now
61
+ if trophy.name == "" && trophy.description == ""
62
+ trophy.name = "Hidden Trophy %i" % hidden_trophy_counter
63
+ trophy.description = "Hidden Trophy"
64
+ hidden_trophy_counter += 1
65
+ end
66
+
67
+ trophy_level = slot.search(".trophyholder .trophyimage").first[:title]
68
+ trophy_levels.each do |level|
69
+ if trophy_level.downcase.include?(level.name.downcase)
70
+ level.trophies << trophy
71
+ end
72
+ end
73
+
74
+ unlock_date = slot.search(".lastTrophyTime").text.strip
75
+
76
+ if unlock_date.length > 0
77
+ trophy_info[trophy] = unlock_date
78
+ else
79
+ trophy_info[trophy] = nil
80
+ end
81
+
82
+ }
83
+ $logger.debug("Found trophies #{trophy_info}")
84
+ return trophy_info
85
+ end
86
+
87
+ protected
88
+
89
+ def request_account_trophy_page
90
+ @account_trophy_page = @agent.get(TROPHY_PAGE_URL % [@username])
91
+ end
92
+
93
+ private
94
+
95
+ end
@@ -0,0 +1,70 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{trophy-scraper}
8
+ s.version = "1.0.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Vince Cima"]
12
+ s.date = %q{2009-10-15}
13
+ s.default_executable = %q{trophy-scraper}
14
+ s.description = %q{Collect and store trophy data for Playstation Network users.}
15
+ s.email = %q{contact@vincecima.com}
16
+ s.executables = ["trophy-scraper"]
17
+ s.extra_rdoc_files = [
18
+ "LICENSE",
19
+ "README.md"
20
+ ]
21
+ s.files = [
22
+ ".document",
23
+ ".gitignore",
24
+ "LICENSE",
25
+ "README.md",
26
+ "Rakefile",
27
+ "SQL",
28
+ "TODO.md",
29
+ "VERSION",
30
+ "bin/trophy-scraper",
31
+ "lib/trophy_scraper.rb",
32
+ "lib/trophy_scraper/controller.rb",
33
+ "lib/trophy_scraper/core_ext.rb",
34
+ "lib/trophy_scraper/data_store.rb",
35
+ "lib/trophy_scraper/models/game.rb",
36
+ "lib/trophy_scraper/models/meta.rb",
37
+ "lib/trophy_scraper/models/trophy.rb",
38
+ "lib/trophy_scraper/models/trophy_level.rb",
39
+ "lib/trophy_scraper/models/unlock.rb",
40
+ "lib/trophy_scraper/models/user.rb",
41
+ "lib/trophy_scraper/models/user_stat.rb",
42
+ "lib/trophy_scraper/scraper.rb",
43
+ "trophy-scraper.gemspec"
44
+ ]
45
+ s.homepage = %q{http://vincecima.com/blog/trophy-scraper/}
46
+ s.rdoc_options = ["--charset=UTF-8"]
47
+ s.require_paths = ["lib"]
48
+ s.rubyforge_project = %q{trophy-scraper}
49
+ s.rubygems_version = %q{1.3.5}
50
+ s.summary = %q{Collect and store trophy data for Playstation Network users.}
51
+
52
+ if s.respond_to? :specification_version then
53
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
54
+ s.specification_version = 3
55
+
56
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
57
+ s.add_runtime_dependency(%q<dm-core>, [">= 0.9.11"])
58
+ s.add_runtime_dependency(%q<do_sqlite3>, [">= 0.9.12"])
59
+ s.add_runtime_dependency(%q<mechanize>, [">= 0.9.3"])
60
+ else
61
+ s.add_dependency(%q<dm-core>, [">= 0.9.11"])
62
+ s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
63
+ s.add_dependency(%q<mechanize>, [">= 0.9.3"])
64
+ end
65
+ else
66
+ s.add_dependency(%q<dm-core>, [">= 0.9.11"])
67
+ s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
68
+ s.add_dependency(%q<mechanize>, [">= 0.9.3"])
69
+ end
70
+ end
metadata ADDED
@@ -0,0 +1,106 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: trophy-scraper
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Vince Cima
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-10-15 00:00:00 -05:00
13
+ default_executable: trophy-scraper
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: dm-core
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 0.9.11
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: do_sqlite3
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.9.12
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: mechanize
37
+ type: :runtime
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: 0.9.3
44
+ version:
45
+ description: Collect and store trophy data for Playstation Network users.
46
+ email: contact@vincecima.com
47
+ executables:
48
+ - trophy-scraper
49
+ extensions: []
50
+
51
+ extra_rdoc_files:
52
+ - LICENSE
53
+ - README.md
54
+ files:
55
+ - .document
56
+ - .gitignore
57
+ - LICENSE
58
+ - README.md
59
+ - Rakefile
60
+ - SQL
61
+ - TODO.md
62
+ - VERSION
63
+ - bin/trophy-scraper
64
+ - lib/trophy_scraper.rb
65
+ - lib/trophy_scraper/controller.rb
66
+ - lib/trophy_scraper/core_ext.rb
67
+ - lib/trophy_scraper/data_store.rb
68
+ - lib/trophy_scraper/models/game.rb
69
+ - lib/trophy_scraper/models/meta.rb
70
+ - lib/trophy_scraper/models/trophy.rb
71
+ - lib/trophy_scraper/models/trophy_level.rb
72
+ - lib/trophy_scraper/models/unlock.rb
73
+ - lib/trophy_scraper/models/user.rb
74
+ - lib/trophy_scraper/models/user_stat.rb
75
+ - lib/trophy_scraper/scraper.rb
76
+ - trophy-scraper.gemspec
77
+ has_rdoc: true
78
+ homepage: http://vincecima.com/blog/trophy-scraper/
79
+ licenses: []
80
+
81
+ post_install_message:
82
+ rdoc_options:
83
+ - --charset=UTF-8
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: "0"
91
+ version:
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: "0"
97
+ version:
98
+ requirements: []
99
+
100
+ rubyforge_project: trophy-scraper
101
+ rubygems_version: 1.3.5
102
+ signing_key:
103
+ specification_version: 3
104
+ summary: Collect and store trophy data for Playstation Network users.
105
+ test_files: []
106
+