trophy-scraper 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
@@ -0,0 +1,6 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
6
+ proj/
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License
2
+
3
+ Copyright (c) 2009 Vince Cima
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,39 @@
1
+ ## TrophyScraper
2
+ ###Collect and store trophy data for Playstation Network users.
3
+
4
+ ## Dependencies
5
+ * Sqlite
6
+ * DataMapper
7
+ * Mechanize
8
+
9
+ ## Installation
10
+ ### Generic
11
+ gem install trophy-scraper
12
+ ### Ubuntu/Debian
13
+ sudo aptitude install ruby ruby-dev rubygems sqlite3 libsqlite3-dev make libopenssl-ruby libxslt-dev libxml2-dev
14
+ sudo gem install trophy-scraper
15
+ The rubygems wrapped exectuable will be located at:
16
+
17
+ /var/lib/gems/1.8/bin/trophy-scraper
18
+ Make trophy-scraper available in the default PATH:
19
+
20
+ sudo ln -s /var/lib/gems/1.8/bin/trophy-scraper /usr/bin/trophy-scraper
21
+
22
+ ## Usage
23
+ ### Adding a user / Update a single user
24
+ trophy-scraper --username "Insert Username Here"
25
+ ### Updating all users
26
+ trophy-scraper
27
+ ### Specifying a custom database location
28
+ trophy-scraper --db "sqlite3://home/joe/trophy_scraper.db"
29
+ ### Changing the logging options
30
+ trophy-scraper --log_level debug --log_dest "~/trophy-scraper.log" --db_log_level error --db_log_dest "~/trophy-scraper-db.log"
31
+ ### Help
32
+ trophy-scraper --help
33
+
34
+ ## Limitations
35
+ 1. Some trophies are marked as "spoilers" and will never show their name and description.
36
+ 1. Some games simply don't show up in the list of games played. Their trophies will contribute to a player's total however.
37
+
38
+ ## Thanks
39
+ Thanks to the Haml and Jekyll projects for a little bit of code and lot of great examples on how to layout a Ruby project.
@@ -0,0 +1,64 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "trophy-scraper"
8
+ gem.summary = %Q{Collect and store trophy data for Playstation Network users.}
9
+ gem.description = %Q{Collect and store trophy data for Playstation Network users.}
10
+ gem.email = "contact@vincecima.com"
11
+ gem.homepage = "http://vincecima.com/blog/trophy-scraper/"
12
+ gem.authors = ["Vince Cima"]
13
+ gem.rubyforge_project = "trophy-scraper"
14
+ gem.add_dependency("dm-core",">= 0.9.11")
15
+ gem.add_dependency("do_sqlite3",">= 0.9.12")
16
+ gem.add_dependency("mechanize",">= 0.9.3")
17
+ gem.rubyforge_project = "trophy-scraper"
18
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
19
+ end
20
+ Jeweler::GemcutterTasks.new
21
+ Jeweler::RubyforgeTasks.new do |rubyforge|
22
+ rubyforge.doc_task = "rdoc"
23
+ end
24
+ rescue LoadError
25
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
26
+ end
27
+
28
+ require 'rake/testtask'
29
+ Rake::TestTask.new(:test) do |test|
30
+ test.libs << 'lib' << 'test'
31
+ test.pattern = 'test/**/*_test.rb'
32
+ test.verbose = true
33
+ end
34
+
35
+ begin
36
+ require 'rcov/rcovtask'
37
+ Rcov::RcovTask.new do |test|
38
+ test.libs << 'test'
39
+ test.pattern = 'test/**/*_test.rb'
40
+ test.verbose = true
41
+ end
42
+ rescue LoadError
43
+ task :rcov do
44
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
45
+ end
46
+ end
47
+
48
+ task :test => :check_dependencies
49
+
50
+ task :default => :test
51
+
52
+ require 'rake/rdoctask'
53
+ Rake::RDocTask.new do |rdoc|
54
+ if File.exist?('VERSION')
55
+ version = File.read('VERSION')
56
+ else
57
+ version = ""
58
+ end
59
+
60
+ rdoc.rdoc_dir = 'rdoc'
61
+ rdoc.title = "trophy-scraper #{version}"
62
+ rdoc.rdoc_files.include('README*')
63
+ rdoc.rdoc_files.include('lib/**/*.rb')
64
+ end
data/SQL ADDED
@@ -0,0 +1,6 @@
1
+ SELECT g.title, COUNT(g.title)
2
+ FROM unlocks u, trophies t, games g
3
+ WHERE user_username = 'username'
4
+ AND u.trophy_id = t.id
5
+ AND t.game_id = g.id
6
+ GROUP BY g.title
data/TODO.md ADDED
@@ -0,0 +1,10 @@
1
+ ## User Management
2
+ 1. Add the ability to remove a user from the data store.
3
+ 1. Add/Remove groups of users without triggering an update.
4
+
5
+ ## Trophy Handling
6
+ 1. Detect trophies that are displayed on Playstation.com and log out message.
7
+ 1. Better handle "spoiler" trophies.
8
+
9
+ ## Scraper
10
+ 1. Add detection of page structure changes.
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.0.0
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
4
+
5
+ help = <<HELP
6
+ TrophyScraper downloads trophy information for Playstation Network Accounts.
7
+
8
+ Basic Command Line Usage:
9
+ trophy-scraper # Update all users in database.
10
+ trophy-scraper --username "username" # Update "username".
11
+ trophy-scraper --db "sqlite3::memory:" # DataMapper db string.
12
+
13
+ Configuration is read from '~/.trophy_scraper.yml' but can be overriden
14
+ using the following options:
15
+
16
+ HELP
17
+
18
+ require 'optparse'
19
+ require 'trophy_scraper'
20
+
21
+ options = {}
22
+ opts = OptionParser.new do |opts|
23
+ opts.banner = help
24
+
25
+ opts.on("--config [PATH]", "Full path to yml configuration file.") do |path|
26
+ options['config'] = path
27
+ end
28
+
29
+ opts.on("--db [DB_STRING]", "DataMapper database string.") do |db_string|
30
+ options['db'] = db_string
31
+ end
32
+
33
+ opts.on("--username [USERNAME]", "Username to run the trophy update for.") do |username|
34
+ options['username'] = username
35
+ end
36
+
37
+ opts.on("--log-level [LOG_LEVEL]", "Standard Ruby logger output level.") do |log_level|
38
+ if log_level.downcase == "debug"
39
+ options['log_level'] = Logger::DEBUG
40
+ elsif log_level.downcase == "info"
41
+ options['log_level'] = Logger::INFO
42
+ elsif log_level.downcase == "warn"
43
+ options['log_level'] = Logger::WARN
44
+ elsif log_level.downcase == "error"
45
+ options['log_level'] = Logger::ERROR
46
+ elsif log_level.downcase == "fatal"
47
+ options['log_level'] = Logger::FATAL
48
+ end
49
+ end
50
+
51
+ opts.on("--log-dest [LOG_DEST]", "STDOUT, STDERR or full path to log file.") do |log_dest|
52
+ if log_dest.downcase == "STDOUT".downcase
53
+ options['log_dest'] = STDOUT
54
+ elsif log_dest.downcase == "STDERR".downcase
55
+ options['log_dest'] = STDERR
56
+ else
57
+ options['log_dest'] = File.expand_path(log_dest)
58
+ end
59
+ end
60
+
61
+ opts.on("--db-log-level [LOG_LEVEL]", "Ruby logger output level for database.") do |log_level|
62
+ if log_level.downcase == "off"
63
+ options['db_log_level'] = :off
64
+ elsif log_level.downcase == "debug"
65
+ options['db_log_level'] = :debug
66
+ elsif log_level.downcase == "info"
67
+ options['db_log_level'] = :info
68
+ elsif log_level.downcase == "warn"
69
+ options['db_log_level'] = :warn
70
+ elsif log_level.downcase == "error"
71
+ options['db_log_level'] = :error
72
+ elsif log_level.downcase == "fatal"
73
+ options['db_log_level'] = :fatal
74
+ end
75
+ end
76
+
77
+ opts.on("--db-log-dest [LOG_DEST]", "STDOUT, STDERR or full path to db log file.") do |db_log_dest|
78
+ if db_log_dest.downcase == "STDOUT".downcase
79
+ options['db_log_dest'] = STDOUT
80
+ elsif db_log_dest.downcase == "STDERR".downcase
81
+ options['db_log_dest'] = STDERR
82
+ else
83
+ options['db_log_dest'] = File.expand_path(db_log_dest)
84
+ end
85
+ end
86
+ end
87
+
88
+ # Read command line options into `options` hash
89
+ opts.parse!
90
+
91
+ options = TrophyScraper.configuration(options)
92
+
93
+ controller = Controller.new(options)
94
+ controller.process
@@ -0,0 +1,68 @@
1
+ $:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
2
+
3
+ # core
4
+
5
+ # stdlib
6
+ require "logger"
7
+
8
+ # 3rd party
9
+ require 'dm-core'
10
+ require 'mechanize'
11
+
12
+ # internal requires
13
+ require "trophy_scraper/controller"
14
+ require "trophy_scraper/core_ext"
15
+ require "trophy_scraper/data_store"
16
+ require "trophy_scraper/scraper"
17
+ require "trophy_scraper/models/meta"
18
+ require "trophy_scraper/models/user"
19
+ require "trophy_scraper/models/user_stat"
20
+ require "trophy_scraper/models/unlock"
21
+ require "trophy_scraper/models/trophy"
22
+ require "trophy_scraper/models/game"
23
+ require "trophy_scraper/models/trophy_level"
24
+
25
+ module TrophyScraper
26
+ # Default options. Overriden by values in _config.yml or command-line opts.
27
+ # (Strings rather symbols used for compatability with YAML)
28
+ DEFAULTS = {
29
+ 'config' => '~/.trophy_scraper.yml',
30
+ 'db' => '',
31
+ 'log_level' => Logger::INFO,
32
+ 'log_dest' => STDOUT,
33
+ 'db' => "sqlite3://%s/trophy_scraper.db" % File.expand_path("~"),
34
+ 'db_log_level' => "fatal",
35
+ 'db_log_dest' => STDOUT
36
+ }
37
+
38
+ # Generate a TrophyScraper configuration Hash by merging the default options
39
+ # with anything in config.yml, and adding the given options on top
40
+ # +override+ is a Hash of config directives
41
+ #
42
+ # Returns Hash
43
+ def self.configuration(override)
44
+ # config.yml may override default source location, but until
45
+ # then, we need to know where to look for config.yml
46
+ source = override['config'] || TrophyScraper::DEFAULTS['config']
47
+
48
+ # Get configuration from <source>
49
+ config_file = File.expand_path(source)
50
+ begin
51
+ config = YAML.load_file(config_file)
52
+ raise "Invalid configuration - #{config_file}" if !config.is_a?(Hash)
53
+ STDOUT.puts "Configuration from #{config_file}"
54
+ rescue => err
55
+ STDERR.puts "WARNING: Could not read configuration. Using defaults (and options)."
56
+ STDERR.puts "\t" + err.to_s
57
+ config = {}
58
+ end
59
+
60
+ # Merge DEFAULTS < config.yml < override
61
+ TrophyScraper::DEFAULTS.deep_merge(config).deep_merge(override)
62
+ end
63
+
64
+ def self.version
65
+ yml = YAML.load(File.read(File.join(File.dirname(__FILE__), *%w[.. VERSION.yml])))
66
+ "#{yml[:major]}.#{yml[:minor]}.#{yml[:patch]}"
67
+ end
68
+ end
@@ -0,0 +1,91 @@
1
+ class Controller
2
+ def initialize(options)
3
+ @options = options.clone
4
+ $logger = Logger.new(@options['log_dest'])
5
+ $logger.level = @options['log_level']
6
+ end
7
+
8
+ def process
9
+ #Initialize connection to db and do any version migration needed.
10
+ data_store = DataStore.new(@options['db'], @options['db_log_level'], @options['db_log_dest'])
11
+
12
+ users_to_check = get_users_to_check
13
+ trophy_levels = TrophyLevel.all
14
+
15
+ users_to_check.each do |user|
16
+ $logger.info("Preparing to update #{user.username}.")
17
+ scraper = Scraper.new(user.username)
18
+
19
+ user_summary_stats = scraper.get_user_summary_stats
20
+ if user_summary_stats.total_trophies > user.unlocks.size
21
+ $logger.info("Website reports more trophies for #{user.username} than data store.")
22
+ user.user_stats << user_summary_stats
23
+
24
+ games_summary_stats = scraper.get_game_summary_stats()
25
+ games_summary_stats.each_pair do |found_game, trophies_earned|
26
+ game = Game.first(:title => found_game.title)
27
+
28
+ if game == nil
29
+ game = found_game
30
+
31
+ game.save
32
+ $logger.debug("Saved new game: #{game.title}")
33
+ end
34
+
35
+ trophy_info = scraper.get_game_detail_stats(found_game.uri, trophy_levels)
36
+ trophy_info.each_pair do |found_trophy, earned_date|
37
+ trophy = game.trophies.first(:name => found_trophy.name, :game_id => game.id, :trophy_level_id => found_trophy.trophy_level_id)
38
+ if trophy == nil
39
+ trophy = found_trophy
40
+
41
+ game.trophies << trophy
42
+ trophy.save
43
+ $logger.debug("Saved new trophy: #{trophy.name}")
44
+ end
45
+
46
+ if earned_date != nil
47
+ unlock = Unlock.first(:user_username => user.username, :trophy_id => trophy.id)
48
+ if unlock == nil
49
+ unlock = Unlock.new(:earned_on => earned_date)
50
+ trophy.unlocks << unlock
51
+ user.unlocks << unlock
52
+ unlock.save
53
+ $logger.info("#{user.username} has unlocked '#{trophy.name}' in '#{game.title}'.")
54
+ end
55
+ end
56
+ end
57
+ end
58
+ elsif user_summary_stats.total_trophies == user.unlocks.size
59
+ $logger.info("Same number of trophies on website and in data store. Nothing to update for #{user.username}")
60
+ else
61
+ $logger.warn("Website reports less trophies than data store for #{user.username}. Something is wrong.")
62
+ end
63
+
64
+ user.last_updated = Time.now
65
+ user.save
66
+ end
67
+
68
+ $logger.close
69
+ end
70
+
71
+ private
72
+
73
+ def get_users_to_check
74
+ users_to_check = Array.new
75
+ if @options['username'] != nil
76
+ user = User.get(@options['username'])
77
+ if user == nil
78
+ user = User.new(:username => @options['username'])
79
+ user.save
80
+ $logger.info("#{user.username} is a new user.")
81
+ end
82
+ users_to_check.push(user)
83
+ $logger.info("#{user.username} will be the only user checked.")
84
+ else
85
+ users_to_check = User.all
86
+ $logger.info("All users in data store will be checked.")
87
+ end
88
+
89
+ return users_to_check
90
+ end
91
+ end
@@ -0,0 +1,22 @@
1
+ class Hash
2
+ # Merges self with another hash, recursively.
3
+ #
4
+ # This code was lovingly stolen from some random gem:
5
+ # http://gemjack.com/gems/tartan-0.1.1/classes/Hash.html
6
+ #
7
+ # Thanks to whoever made it.
8
+ def deep_merge(hash)
9
+ target = dup
10
+
11
+ hash.keys.each do |key|
12
+ if hash[key].is_a? Hash and self[key].is_a? Hash
13
+ target[key] = target[key].deep_merge(hash[key])
14
+ next
15
+ end
16
+
17
+ target[key] = hash[key]
18
+ end
19
+
20
+ target
21
+ end
22
+ end
@@ -0,0 +1,40 @@
1
+ class DataStore
2
+ @@DB_VERSION = 1
3
+ def initialize(connection_string, logger_level, logger_dest)
4
+ DataMapper::Logger.new(logger_dest, logger_level)
5
+ DataMapper.setup(:default, connection_string)
6
+
7
+ begin
8
+ version_meta = Meta.get("version")
9
+ rescue
10
+ #TODO, only eat the table not found error
11
+ end
12
+ if version_meta == nil
13
+ $logger.info("Database is empty, creating schema.")
14
+ DataMapper.auto_migrate!
15
+ version_meta = Meta.new(:name => "version", :value => @@DB_VERSION)
16
+ version_meta.save
17
+ insert_reference_data
18
+ elsif version_meta.value.to_i < @@DB_VERSION
19
+ $logger.info("Database schema version is older than code's, gracefully updating.")
20
+ DataMapper.auto_upgrade!
21
+ version_meta.value = @@DB_VERSION
22
+ version_meta.save
23
+ elsif version_meta.value.to_i > @@DB_VERSION
24
+ #TODO Code is older than db, warn and abort.
25
+ $logger.warn("Database schema version is newer than code's, aborting.")
26
+ else
27
+ $logger.info("Database schema is up to date.")
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def insert_reference_data
34
+ TrophyLevel.new(:name => "Platinum").save
35
+ TrophyLevel.new(:name => "Gold").save
36
+ TrophyLevel.new(:name => "Silver").save
37
+ TrophyLevel.new(:name => "Bronze").save
38
+ TrophyLevel.new(:name => "Hidden").save
39
+ end
40
+ end
@@ -0,0 +1,11 @@
1
+ class Game
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :title, String, :nullable => false
6
+ property :logo, Object
7
+
8
+ has n, :trophies
9
+
10
+ attr_accessor :uri, :logo_uri
11
+ end
@@ -0,0 +1,6 @@
1
+ class Meta
2
+ include DataMapper::Resource
3
+
4
+ property :name, String, :key => true
5
+ property :value, String
6
+ end
@@ -0,0 +1,19 @@
1
+ class Trophy
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :name, String, :nullable => false
6
+ property :description, String, :nullable => false
7
+ property :icon, Object
8
+
9
+ has n, :unlocks
10
+ has n, :users, :through => :unlocks
11
+ belongs_to :game
12
+ belongs_to :trophy_level
13
+
14
+ attr_accessor :icon_uri
15
+
16
+ def to_s
17
+ return "id = #{@id}, name = #{@name}, description = #{@description}"
18
+ end
19
+ end
@@ -0,0 +1,7 @@
1
+ class TrophyLevel
2
+ include DataMapper::Resource
3
+ property :id, Serial
4
+ property :name, String, :nullable => false
5
+
6
+ has n, :trophies
7
+ end
@@ -0,0 +1,8 @@
1
+ class Unlock
2
+ include DataMapper::Resource
3
+ property :id, Serial
4
+ property :earned_on, DateTime, :nullable => false
5
+
6
+ belongs_to :trophy
7
+ belongs_to :user
8
+ end
@@ -0,0 +1,10 @@
1
+ class User
2
+ include DataMapper::Resource
3
+
4
+ property :username, String, :key => true
5
+ property :last_updated, Time
6
+
7
+ has n, :user_stats, :model => "UserStat"
8
+ has n, :unlocks
9
+ has n, :trophies, :through => :unlocks
10
+ end
@@ -0,0 +1,21 @@
1
+ class UserStat
2
+ include DataMapper::Resource
3
+
4
+ property :id, Serial
5
+ property :number_of_bronze, Integer
6
+ property :number_of_silver, Integer
7
+ property :number_of_gold, Integer
8
+ property :number_of_platinum, Integer
9
+ property :level, Integer
10
+ property :percent_to_next_level, Integer
11
+
12
+ belongs_to :user
13
+
14
+ def total_trophies
15
+ return @number_of_bronze + @number_of_silver + @number_of_gold + @number_of_platinum
16
+ end
17
+
18
+ def to_s
19
+ return "percent_to_next_level = #{@percent_to_next_level}, level = #{@level}, #bronze = #{@number_of_bronze}, #silver = #{@number_of_silver}, #gold = #{@number_of_gold}, #platinum = #{@number_of_platinum}"
20
+ end
21
+ end
@@ -0,0 +1,95 @@
1
+ class Scraper
2
+ TROPHY_PAGE_URL = "http://profiles.us.playstation.com/playstation/psn/profiles/%s"
3
+ TROPHY_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_trophies_data"
4
+ TROPHY_DETAILED_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_title_details_data"
5
+
6
+ def initialize(username)
7
+ @username = username
8
+ @agent = WWW::Mechanize.new
9
+ @update_date = Time.now
10
+ @account_trophy_page = nil
11
+ self.request_account_trophy_page
12
+ end
13
+
14
+ def get_user_summary_stats
15
+ stats = UserStat.new
16
+ stats.level = @account_trophy_page.search("#leveltext").text.strip
17
+ stats.percent_to_next_level = @account_trophy_page.search(".progresstext").text.strip.delete("%")
18
+ stats.number_of_bronze = @account_trophy_page.search(".podium .bronze").text.strip.split[0]
19
+ stats.number_of_silver = @account_trophy_page.search(".podium .silver").text.strip.split[0]
20
+ stats.number_of_gold = @account_trophy_page.search(".podium .gold").text.strip.split[0]
21
+ stats.number_of_platinum = @account_trophy_page.search(".podium .platinum").text.strip.split[0]
22
+ $logger.debug("Found user stats #{stats}")
23
+ return stats
24
+ end
25
+
26
+ def get_game_summary_stats()
27
+ trophy_data = @agent.get(TROPHY_DATA_URL % [@username])
28
+ summary_stats = Hash.new
29
+
30
+ game_slots = trophy_data.search(".slot")
31
+ game_slots.each { |game_data|
32
+ game = Game.new
33
+ game.title = game_data.search(".titletext a").text.strip
34
+ game.uri = game_data.search(".titletext a").first[:href].strip
35
+ game.logo_uri = game_data.search(".titlelogo img").first[:src].strip
36
+ #TODO Make sure we got the right amount of trophy columns
37
+ trophies_earned = Hash.new
38
+ trophies_earned[:bronze] = game_data.search(".trophycount.normal .trophycontent")[0].text.strip
39
+ trophies_earned[:silver] = game_data.search(".trophycount.normal .trophycontent")[1].text.strip
40
+ trophies_earned[:gold] = game_data.search(".trophycount.normal .trophycontent")[2].text.strip
41
+ trophies_earned[:platinum] = game_data.search(".trophycount.normal .trophycontent")[3].text.strip
42
+
43
+ summary_stats[game] = trophies_earned
44
+ }
45
+ return summary_stats
46
+ end
47
+
48
+ def get_game_detail_stats(game_uri, trophy_levels)
49
+ trophy_info = Hash.new
50
+ hidden_trophy_counter = 1
51
+
52
+ titleId = game_uri.split("/")[-1]
53
+ trophy_data = @agent.post(TROPHY_DETAILED_DATA_URL % [@username], "titleId" => titleId, "sortBy" => "id_asc")
54
+
55
+ trophy_data.search(".slotcontent").each { |slot|
56
+ trophy = Trophy.new
57
+ trophy.name = slot.search(".trophyTitleSortField").text.strip
58
+ trophy.description = slot.search(".subtext").text.strip
59
+
60
+ # Game is a "spoiler trophy", ignore for now
61
+ if trophy.name == "" && trophy.description == ""
62
+ trophy.name = "Hidden Trophy %i" % hidden_trophy_counter
63
+ trophy.description = "Hidden Trophy"
64
+ hidden_trophy_counter += 1
65
+ end
66
+
67
+ trophy_level = slot.search(".trophyholder .trophyimage").first[:title]
68
+ trophy_levels.each do |level|
69
+ if trophy_level.downcase.include?(level.name.downcase)
70
+ level.trophies << trophy
71
+ end
72
+ end
73
+
74
+ unlock_date = slot.search(".lastTrophyTime").text.strip
75
+
76
+ if unlock_date.length > 0
77
+ trophy_info[trophy] = unlock_date
78
+ else
79
+ trophy_info[trophy] = nil
80
+ end
81
+
82
+ }
83
+ $logger.debug("Found trophies #{trophy_info}")
84
+ return trophy_info
85
+ end
86
+
87
+ protected
88
+
89
+ def request_account_trophy_page
90
+ @account_trophy_page = @agent.get(TROPHY_PAGE_URL % [@username])
91
+ end
92
+
93
+ private
94
+
95
+ end
@@ -0,0 +1,70 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{trophy-scraper}
8
+ s.version = "1.0.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Vince Cima"]
12
+ s.date = %q{2009-10-15}
13
+ s.default_executable = %q{trophy-scraper}
14
+ s.description = %q{Collect and store trophy data for Playstation Network users.}
15
+ s.email = %q{contact@vincecima.com}
16
+ s.executables = ["trophy-scraper"]
17
+ s.extra_rdoc_files = [
18
+ "LICENSE",
19
+ "README.md"
20
+ ]
21
+ s.files = [
22
+ ".document",
23
+ ".gitignore",
24
+ "LICENSE",
25
+ "README.md",
26
+ "Rakefile",
27
+ "SQL",
28
+ "TODO.md",
29
+ "VERSION",
30
+ "bin/trophy-scraper",
31
+ "lib/trophy_scraper.rb",
32
+ "lib/trophy_scraper/controller.rb",
33
+ "lib/trophy_scraper/core_ext.rb",
34
+ "lib/trophy_scraper/data_store.rb",
35
+ "lib/trophy_scraper/models/game.rb",
36
+ "lib/trophy_scraper/models/meta.rb",
37
+ "lib/trophy_scraper/models/trophy.rb",
38
+ "lib/trophy_scraper/models/trophy_level.rb",
39
+ "lib/trophy_scraper/models/unlock.rb",
40
+ "lib/trophy_scraper/models/user.rb",
41
+ "lib/trophy_scraper/models/user_stat.rb",
42
+ "lib/trophy_scraper/scraper.rb",
43
+ "trophy-scraper.gemspec"
44
+ ]
45
+ s.homepage = %q{http://vincecima.com/blog/trophy-scraper/}
46
+ s.rdoc_options = ["--charset=UTF-8"]
47
+ s.require_paths = ["lib"]
48
+ s.rubyforge_project = %q{trophy-scraper}
49
+ s.rubygems_version = %q{1.3.5}
50
+ s.summary = %q{Collect and store trophy data for Playstation Network users.}
51
+
52
+ if s.respond_to? :specification_version then
53
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
54
+ s.specification_version = 3
55
+
56
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
57
+ s.add_runtime_dependency(%q<dm-core>, [">= 0.9.11"])
58
+ s.add_runtime_dependency(%q<do_sqlite3>, [">= 0.9.12"])
59
+ s.add_runtime_dependency(%q<mechanize>, [">= 0.9.3"])
60
+ else
61
+ s.add_dependency(%q<dm-core>, [">= 0.9.11"])
62
+ s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
63
+ s.add_dependency(%q<mechanize>, [">= 0.9.3"])
64
+ end
65
+ else
66
+ s.add_dependency(%q<dm-core>, [">= 0.9.11"])
67
+ s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
68
+ s.add_dependency(%q<mechanize>, [">= 0.9.3"])
69
+ end
70
+ end
metadata ADDED
@@ -0,0 +1,106 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: trophy-scraper
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Vince Cima
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-10-15 00:00:00 -05:00
13
+ default_executable: trophy-scraper
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: dm-core
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: 0.9.11
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: do_sqlite3
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.9.12
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: mechanize
37
+ type: :runtime
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: 0.9.3
44
+ version:
45
+ description: Collect and store trophy data for Playstation Network users.
46
+ email: contact@vincecima.com
47
+ executables:
48
+ - trophy-scraper
49
+ extensions: []
50
+
51
+ extra_rdoc_files:
52
+ - LICENSE
53
+ - README.md
54
+ files:
55
+ - .document
56
+ - .gitignore
57
+ - LICENSE
58
+ - README.md
59
+ - Rakefile
60
+ - SQL
61
+ - TODO.md
62
+ - VERSION
63
+ - bin/trophy-scraper
64
+ - lib/trophy_scraper.rb
65
+ - lib/trophy_scraper/controller.rb
66
+ - lib/trophy_scraper/core_ext.rb
67
+ - lib/trophy_scraper/data_store.rb
68
+ - lib/trophy_scraper/models/game.rb
69
+ - lib/trophy_scraper/models/meta.rb
70
+ - lib/trophy_scraper/models/trophy.rb
71
+ - lib/trophy_scraper/models/trophy_level.rb
72
+ - lib/trophy_scraper/models/unlock.rb
73
+ - lib/trophy_scraper/models/user.rb
74
+ - lib/trophy_scraper/models/user_stat.rb
75
+ - lib/trophy_scraper/scraper.rb
76
+ - trophy-scraper.gemspec
77
+ has_rdoc: true
78
+ homepage: http://vincecima.com/blog/trophy-scraper/
79
+ licenses: []
80
+
81
+ post_install_message:
82
+ rdoc_options:
83
+ - --charset=UTF-8
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: "0"
91
+ version:
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: "0"
97
+ version:
98
+ requirements: []
99
+
100
+ rubyforge_project: trophy-scraper
101
+ rubygems_version: 1.3.5
102
+ signing_key:
103
+ specification_version: 3
104
+ summary: Collect and store trophy data for Playstation Network users.
105
+ test_files: []
106
+