trophy-scraper 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +6 -0
- data/LICENSE +21 -0
- data/README.md +39 -0
- data/Rakefile +64 -0
- data/SQL +6 -0
- data/TODO.md +10 -0
- data/VERSION +1 -0
- data/bin/trophy-scraper +94 -0
- data/lib/trophy_scraper.rb +68 -0
- data/lib/trophy_scraper/controller.rb +91 -0
- data/lib/trophy_scraper/core_ext.rb +22 -0
- data/lib/trophy_scraper/data_store.rb +40 -0
- data/lib/trophy_scraper/models/game.rb +11 -0
- data/lib/trophy_scraper/models/meta.rb +6 -0
- data/lib/trophy_scraper/models/trophy.rb +19 -0
- data/lib/trophy_scraper/models/trophy_level.rb +7 -0
- data/lib/trophy_scraper/models/unlock.rb +8 -0
- data/lib/trophy_scraper/models/user.rb +10 -0
- data/lib/trophy_scraper/models/user_stat.rb +21 -0
- data/lib/trophy_scraper/scraper.rb +95 -0
- data/trophy-scraper.gemspec +70 -0
- metadata +106 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2009 Vince Cima
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
## TrophyScraper
|
2
|
+
###Collect and store trophy data for Playstation Network users.
|
3
|
+
|
4
|
+
## Dependencies
|
5
|
+
* Sqlite
|
6
|
+
* DataMapper
|
7
|
+
* Mechanize
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
### Generic
|
11
|
+
gem install trophy-scraper
|
12
|
+
### Ubuntu/Debian
|
13
|
+
sudo aptitude install ruby ruby-dev rubygems sqlite3 libsqlite3-dev make libopenssl-ruby libxslt-dev libxml2-dev
|
14
|
+
sudo gem install trophy-scraper
|
15
|
+
The rubygems wrapped exectuable will be located at:
|
16
|
+
|
17
|
+
/var/lib/gems/1.8/bin/trophy-scraper
|
18
|
+
Make trophy-scraper available in the default PATH:
|
19
|
+
|
20
|
+
sudo ln -s /var/lib/gems/1.8/bin/trophy-scraper /usr/bin/trophy-scraper
|
21
|
+
|
22
|
+
## Usage
|
23
|
+
### Adding a user / Update a single user
|
24
|
+
trophy-scraper --username "Insert Username Here"
|
25
|
+
### Updating all users
|
26
|
+
trophy-scraper
|
27
|
+
### Specifying a custom database location
|
28
|
+
trophy-scraper --db "sqlite3://home/joe/trophy_scraper.db"
|
29
|
+
### Changing the logging options
|
30
|
+
trophy-scraper --log_level debug --log_dest "~/trophy-scraper.log" --db_log_level error --db_log_dest "~/trophy-scraper-db.log"
|
31
|
+
### Help
|
32
|
+
trophy-scraper --help
|
33
|
+
|
34
|
+
## Limitations
|
35
|
+
1. Some trophies are marked as "spoilers" and will never show their name and description.
|
36
|
+
1. Some games simply don't show up in the list of games played. Their trophies will contribute to a player's total however.
|
37
|
+
|
38
|
+
## Thanks
|
39
|
+
Thanks to the Haml and Jekyll projects for a little bit of code and lot of great examples on how to layout a Ruby project.
|
data/Rakefile
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "trophy-scraper"
|
8
|
+
gem.summary = %Q{Collect and store trophy data for Playstation Network users.}
|
9
|
+
gem.description = %Q{Collect and store trophy data for Playstation Network users.}
|
10
|
+
gem.email = "contact@vincecima.com"
|
11
|
+
gem.homepage = "http://vincecima.com/blog/trophy-scraper/"
|
12
|
+
gem.authors = ["Vince Cima"]
|
13
|
+
gem.rubyforge_project = "trophy-scraper"
|
14
|
+
gem.add_dependency("dm-core",">= 0.9.11")
|
15
|
+
gem.add_dependency("do_sqlite3",">= 0.9.12")
|
16
|
+
gem.add_dependency("mechanize",">= 0.9.3")
|
17
|
+
gem.rubyforge_project = "trophy-scraper"
|
18
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
19
|
+
end
|
20
|
+
Jeweler::GemcutterTasks.new
|
21
|
+
Jeweler::RubyforgeTasks.new do |rubyforge|
|
22
|
+
rubyforge.doc_task = "rdoc"
|
23
|
+
end
|
24
|
+
rescue LoadError
|
25
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
26
|
+
end
|
27
|
+
|
28
|
+
require 'rake/testtask'
|
29
|
+
Rake::TestTask.new(:test) do |test|
|
30
|
+
test.libs << 'lib' << 'test'
|
31
|
+
test.pattern = 'test/**/*_test.rb'
|
32
|
+
test.verbose = true
|
33
|
+
end
|
34
|
+
|
35
|
+
begin
|
36
|
+
require 'rcov/rcovtask'
|
37
|
+
Rcov::RcovTask.new do |test|
|
38
|
+
test.libs << 'test'
|
39
|
+
test.pattern = 'test/**/*_test.rb'
|
40
|
+
test.verbose = true
|
41
|
+
end
|
42
|
+
rescue LoadError
|
43
|
+
task :rcov do
|
44
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
task :test => :check_dependencies
|
49
|
+
|
50
|
+
task :default => :test
|
51
|
+
|
52
|
+
require 'rake/rdoctask'
|
53
|
+
Rake::RDocTask.new do |rdoc|
|
54
|
+
if File.exist?('VERSION')
|
55
|
+
version = File.read('VERSION')
|
56
|
+
else
|
57
|
+
version = ""
|
58
|
+
end
|
59
|
+
|
60
|
+
rdoc.rdoc_dir = 'rdoc'
|
61
|
+
rdoc.title = "trophy-scraper #{version}"
|
62
|
+
rdoc.rdoc_files.include('README*')
|
63
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
64
|
+
end
|
data/SQL
ADDED
data/TODO.md
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
## User Management
|
2
|
+
1. Add the ability to remove a user from the data store.
|
3
|
+
1. Add/Remove groups of users without triggering an update.
|
4
|
+
|
5
|
+
## Trophy Handling
|
6
|
+
1. Detect trophies that are displayed on Playstation.com and log out message.
|
7
|
+
1. Better handle "spoiler" trophies.
|
8
|
+
|
9
|
+
## Scraper
|
10
|
+
1. Add detection of page structure changes.
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.0.0
|
data/bin/trophy-scraper
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
|
4
|
+
|
5
|
+
help = <<HELP
|
6
|
+
TrophyScraper downloads trophy information for Playstation Network Accounts.
|
7
|
+
|
8
|
+
Basic Command Line Usage:
|
9
|
+
trophy-scraper # Update all users in database.
|
10
|
+
trophy-scraper --username "username" # Update "username".
|
11
|
+
trophy-scraper --db "sqlite3::memory:" # DataMapper db string.
|
12
|
+
|
13
|
+
Configuration is read from '~/.trophy_scraper.yml' but can be overriden
|
14
|
+
using the following options:
|
15
|
+
|
16
|
+
HELP
|
17
|
+
|
18
|
+
require 'optparse'
|
19
|
+
require 'trophy_scraper'
|
20
|
+
|
21
|
+
options = {}
|
22
|
+
opts = OptionParser.new do |opts|
|
23
|
+
opts.banner = help
|
24
|
+
|
25
|
+
opts.on("--config [PATH]", "Full path to yml configuration file.") do |path|
|
26
|
+
options['config'] = path
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on("--db [DB_STRING]", "DataMapper database string.") do |db_string|
|
30
|
+
options['db'] = db_string
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on("--username [USERNAME]", "Username to run the trophy update for.") do |username|
|
34
|
+
options['username'] = username
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.on("--log-level [LOG_LEVEL]", "Standard Ruby logger output level.") do |log_level|
|
38
|
+
if log_level.downcase == "debug"
|
39
|
+
options['log_level'] = Logger::DEBUG
|
40
|
+
elsif log_level.downcase == "info"
|
41
|
+
options['log_level'] = Logger::INFO
|
42
|
+
elsif log_level.downcase == "warn"
|
43
|
+
options['log_level'] = Logger::WARN
|
44
|
+
elsif log_level.downcase == "error"
|
45
|
+
options['log_level'] = Logger::ERROR
|
46
|
+
elsif log_level.downcase == "fatal"
|
47
|
+
options['log_level'] = Logger::FATAL
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.on("--log-dest [LOG_DEST]", "STDOUT, STDERR or full path to log file.") do |log_dest|
|
52
|
+
if log_dest.downcase == "STDOUT".downcase
|
53
|
+
options['log_dest'] = STDOUT
|
54
|
+
elsif log_dest.downcase == "STDERR".downcase
|
55
|
+
options['log_dest'] = STDERR
|
56
|
+
else
|
57
|
+
options['log_dest'] = File.expand_path(log_dest)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.on("--db-log-level [LOG_LEVEL]", "Ruby logger output level for database.") do |log_level|
|
62
|
+
if log_level.downcase == "off"
|
63
|
+
options['db_log_level'] = :off
|
64
|
+
elsif log_level.downcase == "debug"
|
65
|
+
options['db_log_level'] = :debug
|
66
|
+
elsif log_level.downcase == "info"
|
67
|
+
options['db_log_level'] = :info
|
68
|
+
elsif log_level.downcase == "warn"
|
69
|
+
options['db_log_level'] = :warn
|
70
|
+
elsif log_level.downcase == "error"
|
71
|
+
options['db_log_level'] = :error
|
72
|
+
elsif log_level.downcase == "fatal"
|
73
|
+
options['db_log_level'] = :fatal
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
opts.on("--db-log-dest [LOG_DEST]", "STDOUT, STDERR or full path to db log file.") do |db_log_dest|
|
78
|
+
if db_log_dest.downcase == "STDOUT".downcase
|
79
|
+
options['db_log_dest'] = STDOUT
|
80
|
+
elsif db_log_dest.downcase == "STDERR".downcase
|
81
|
+
options['db_log_dest'] = STDERR
|
82
|
+
else
|
83
|
+
options['db_log_dest'] = File.expand_path(db_log_dest)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# Read command line options into `options` hash
|
89
|
+
opts.parse!
|
90
|
+
|
91
|
+
options = TrophyScraper.configuration(options)
|
92
|
+
|
93
|
+
controller = Controller.new(options)
|
94
|
+
controller.process
|
@@ -0,0 +1,68 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
|
2
|
+
|
3
|
+
# core
|
4
|
+
|
5
|
+
# stdlib
|
6
|
+
require "logger"
|
7
|
+
|
8
|
+
# 3rd party
|
9
|
+
require 'dm-core'
|
10
|
+
require 'mechanize'
|
11
|
+
|
12
|
+
# internal requires
|
13
|
+
require "trophy_scraper/controller"
|
14
|
+
require "trophy_scraper/core_ext"
|
15
|
+
require "trophy_scraper/data_store"
|
16
|
+
require "trophy_scraper/scraper"
|
17
|
+
require "trophy_scraper/models/meta"
|
18
|
+
require "trophy_scraper/models/user"
|
19
|
+
require "trophy_scraper/models/user_stat"
|
20
|
+
require "trophy_scraper/models/unlock"
|
21
|
+
require "trophy_scraper/models/trophy"
|
22
|
+
require "trophy_scraper/models/game"
|
23
|
+
require "trophy_scraper/models/trophy_level"
|
24
|
+
|
25
|
+
module TrophyScraper
|
26
|
+
# Default options. Overriden by values in _config.yml or command-line opts.
|
27
|
+
# (Strings rather symbols used for compatability with YAML)
|
28
|
+
DEFAULTS = {
|
29
|
+
'config' => '~/.trophy_scraper.yml',
|
30
|
+
'db' => '',
|
31
|
+
'log_level' => Logger::INFO,
|
32
|
+
'log_dest' => STDOUT,
|
33
|
+
'db' => "sqlite3://%s/trophy_scraper.db" % File.expand_path("~"),
|
34
|
+
'db_log_level' => "fatal",
|
35
|
+
'db_log_dest' => STDOUT
|
36
|
+
}
|
37
|
+
|
38
|
+
# Generate a TrophyScraper configuration Hash by merging the default options
|
39
|
+
# with anything in config.yml, and adding the given options on top
|
40
|
+
# +override+ is a Hash of config directives
|
41
|
+
#
|
42
|
+
# Returns Hash
|
43
|
+
def self.configuration(override)
|
44
|
+
# config.yml may override default source location, but until
|
45
|
+
# then, we need to know where to look for config.yml
|
46
|
+
source = override['config'] || TrophyScraper::DEFAULTS['config']
|
47
|
+
|
48
|
+
# Get configuration from <source>
|
49
|
+
config_file = File.expand_path(source)
|
50
|
+
begin
|
51
|
+
config = YAML.load_file(config_file)
|
52
|
+
raise "Invalid configuration - #{config_file}" if !config.is_a?(Hash)
|
53
|
+
STDOUT.puts "Configuration from #{config_file}"
|
54
|
+
rescue => err
|
55
|
+
STDERR.puts "WARNING: Could not read configuration. Using defaults (and options)."
|
56
|
+
STDERR.puts "\t" + err.to_s
|
57
|
+
config = {}
|
58
|
+
end
|
59
|
+
|
60
|
+
# Merge DEFAULTS < config.yml < override
|
61
|
+
TrophyScraper::DEFAULTS.deep_merge(config).deep_merge(override)
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.version
|
65
|
+
yml = YAML.load(File.read(File.join(File.dirname(__FILE__), *%w[.. VERSION.yml])))
|
66
|
+
"#{yml[:major]}.#{yml[:minor]}.#{yml[:patch]}"
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
class Controller
|
2
|
+
def initialize(options)
|
3
|
+
@options = options.clone
|
4
|
+
$logger = Logger.new(@options['log_dest'])
|
5
|
+
$logger.level = @options['log_level']
|
6
|
+
end
|
7
|
+
|
8
|
+
def process
|
9
|
+
#Initialize connection to db and do any version migration needed.
|
10
|
+
data_store = DataStore.new(@options['db'], @options['db_log_level'], @options['db_log_dest'])
|
11
|
+
|
12
|
+
users_to_check = get_users_to_check
|
13
|
+
trophy_levels = TrophyLevel.all
|
14
|
+
|
15
|
+
users_to_check.each do |user|
|
16
|
+
$logger.info("Preparing to update #{user.username}.")
|
17
|
+
scraper = Scraper.new(user.username)
|
18
|
+
|
19
|
+
user_summary_stats = scraper.get_user_summary_stats
|
20
|
+
if user_summary_stats.total_trophies > user.unlocks.size
|
21
|
+
$logger.info("Website reports more trophies for #{user.username} than data store.")
|
22
|
+
user.user_stats << user_summary_stats
|
23
|
+
|
24
|
+
games_summary_stats = scraper.get_game_summary_stats()
|
25
|
+
games_summary_stats.each_pair do |found_game, trophies_earned|
|
26
|
+
game = Game.first(:title => found_game.title)
|
27
|
+
|
28
|
+
if game == nil
|
29
|
+
game = found_game
|
30
|
+
|
31
|
+
game.save
|
32
|
+
$logger.debug("Saved new game: #{game.title}")
|
33
|
+
end
|
34
|
+
|
35
|
+
trophy_info = scraper.get_game_detail_stats(found_game.uri, trophy_levels)
|
36
|
+
trophy_info.each_pair do |found_trophy, earned_date|
|
37
|
+
trophy = game.trophies.first(:name => found_trophy.name, :game_id => game.id, :trophy_level_id => found_trophy.trophy_level_id)
|
38
|
+
if trophy == nil
|
39
|
+
trophy = found_trophy
|
40
|
+
|
41
|
+
game.trophies << trophy
|
42
|
+
trophy.save
|
43
|
+
$logger.debug("Saved new trophy: #{trophy.name}")
|
44
|
+
end
|
45
|
+
|
46
|
+
if earned_date != nil
|
47
|
+
unlock = Unlock.first(:user_username => user.username, :trophy_id => trophy.id)
|
48
|
+
if unlock == nil
|
49
|
+
unlock = Unlock.new(:earned_on => earned_date)
|
50
|
+
trophy.unlocks << unlock
|
51
|
+
user.unlocks << unlock
|
52
|
+
unlock.save
|
53
|
+
$logger.info("#{user.username} has unlocked '#{trophy.name}' in '#{game.title}'.")
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
elsif user_summary_stats.total_trophies == user.unlocks.size
|
59
|
+
$logger.info("Same number of trophies on website and in data store. Nothing to update for #{user.username}")
|
60
|
+
else
|
61
|
+
$logger.warn("Website reports less trophies than data store for #{user.username}. Something is wrong.")
|
62
|
+
end
|
63
|
+
|
64
|
+
user.last_updated = Time.now
|
65
|
+
user.save
|
66
|
+
end
|
67
|
+
|
68
|
+
$logger.close
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def get_users_to_check
|
74
|
+
users_to_check = Array.new
|
75
|
+
if @options['username'] != nil
|
76
|
+
user = User.get(@options['username'])
|
77
|
+
if user == nil
|
78
|
+
user = User.new(:username => @options['username'])
|
79
|
+
user.save
|
80
|
+
$logger.info("#{user.username} is a new user.")
|
81
|
+
end
|
82
|
+
users_to_check.push(user)
|
83
|
+
$logger.info("#{user.username} will be the only user checked.")
|
84
|
+
else
|
85
|
+
users_to_check = User.all
|
86
|
+
$logger.info("All users in data store will be checked.")
|
87
|
+
end
|
88
|
+
|
89
|
+
return users_to_check
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class Hash
|
2
|
+
# Merges self with another hash, recursively.
|
3
|
+
#
|
4
|
+
# This code was lovingly stolen from some random gem:
|
5
|
+
# http://gemjack.com/gems/tartan-0.1.1/classes/Hash.html
|
6
|
+
#
|
7
|
+
# Thanks to whoever made it.
|
8
|
+
def deep_merge(hash)
|
9
|
+
target = dup
|
10
|
+
|
11
|
+
hash.keys.each do |key|
|
12
|
+
if hash[key].is_a? Hash and self[key].is_a? Hash
|
13
|
+
target[key] = target[key].deep_merge(hash[key])
|
14
|
+
next
|
15
|
+
end
|
16
|
+
|
17
|
+
target[key] = hash[key]
|
18
|
+
end
|
19
|
+
|
20
|
+
target
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
class DataStore
|
2
|
+
@@DB_VERSION = 1
|
3
|
+
def initialize(connection_string, logger_level, logger_dest)
|
4
|
+
DataMapper::Logger.new(logger_dest, logger_level)
|
5
|
+
DataMapper.setup(:default, connection_string)
|
6
|
+
|
7
|
+
begin
|
8
|
+
version_meta = Meta.get("version")
|
9
|
+
rescue
|
10
|
+
#TODO, only eat the table not found error
|
11
|
+
end
|
12
|
+
if version_meta == nil
|
13
|
+
$logger.info("Database is empty, creating schema.")
|
14
|
+
DataMapper.auto_migrate!
|
15
|
+
version_meta = Meta.new(:name => "version", :value => @@DB_VERSION)
|
16
|
+
version_meta.save
|
17
|
+
insert_reference_data
|
18
|
+
elsif version_meta.value.to_i < @@DB_VERSION
|
19
|
+
$logger.info("Database schema version is older than code's, gracefully updating.")
|
20
|
+
DataMapper.auto_upgrade!
|
21
|
+
version_meta.value = @@DB_VERSION
|
22
|
+
version_meta.save
|
23
|
+
elsif version_meta.value.to_i > @@DB_VERSION
|
24
|
+
#TODO Code is older than db, warn and abort.
|
25
|
+
$logger.warn("Database schema version is newer than code's, aborting.")
|
26
|
+
else
|
27
|
+
$logger.info("Database schema is up to date.")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def insert_reference_data
|
34
|
+
TrophyLevel.new(:name => "Platinum").save
|
35
|
+
TrophyLevel.new(:name => "Gold").save
|
36
|
+
TrophyLevel.new(:name => "Silver").save
|
37
|
+
TrophyLevel.new(:name => "Bronze").save
|
38
|
+
TrophyLevel.new(:name => "Hidden").save
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class Trophy
|
2
|
+
include DataMapper::Resource
|
3
|
+
|
4
|
+
property :id, Serial
|
5
|
+
property :name, String, :nullable => false
|
6
|
+
property :description, String, :nullable => false
|
7
|
+
property :icon, Object
|
8
|
+
|
9
|
+
has n, :unlocks
|
10
|
+
has n, :users, :through => :unlocks
|
11
|
+
belongs_to :game
|
12
|
+
belongs_to :trophy_level
|
13
|
+
|
14
|
+
attr_accessor :icon_uri
|
15
|
+
|
16
|
+
def to_s
|
17
|
+
return "id = #{@id}, name = #{@name}, description = #{@description}"
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
class UserStat
|
2
|
+
include DataMapper::Resource
|
3
|
+
|
4
|
+
property :id, Serial
|
5
|
+
property :number_of_bronze, Integer
|
6
|
+
property :number_of_silver, Integer
|
7
|
+
property :number_of_gold, Integer
|
8
|
+
property :number_of_platinum, Integer
|
9
|
+
property :level, Integer
|
10
|
+
property :percent_to_next_level, Integer
|
11
|
+
|
12
|
+
belongs_to :user
|
13
|
+
|
14
|
+
def total_trophies
|
15
|
+
return @number_of_bronze + @number_of_silver + @number_of_gold + @number_of_platinum
|
16
|
+
end
|
17
|
+
|
18
|
+
def to_s
|
19
|
+
return "percent_to_next_level = #{@percent_to_next_level}, level = #{@level}, #bronze = #{@number_of_bronze}, #silver = #{@number_of_silver}, #gold = #{@number_of_gold}, #platinum = #{@number_of_platinum}"
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
class Scraper
|
2
|
+
TROPHY_PAGE_URL = "http://profiles.us.playstation.com/playstation/psn/profiles/%s"
|
3
|
+
TROPHY_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_trophies_data"
|
4
|
+
TROPHY_DETAILED_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_title_details_data"
|
5
|
+
|
6
|
+
def initialize(username)
|
7
|
+
@username = username
|
8
|
+
@agent = WWW::Mechanize.new
|
9
|
+
@update_date = Time.now
|
10
|
+
@account_trophy_page = nil
|
11
|
+
self.request_account_trophy_page
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_user_summary_stats
|
15
|
+
stats = UserStat.new
|
16
|
+
stats.level = @account_trophy_page.search("#leveltext").text.strip
|
17
|
+
stats.percent_to_next_level = @account_trophy_page.search(".progresstext").text.strip.delete("%")
|
18
|
+
stats.number_of_bronze = @account_trophy_page.search(".podium .bronze").text.strip.split[0]
|
19
|
+
stats.number_of_silver = @account_trophy_page.search(".podium .silver").text.strip.split[0]
|
20
|
+
stats.number_of_gold = @account_trophy_page.search(".podium .gold").text.strip.split[0]
|
21
|
+
stats.number_of_platinum = @account_trophy_page.search(".podium .platinum").text.strip.split[0]
|
22
|
+
$logger.debug("Found user stats #{stats}")
|
23
|
+
return stats
|
24
|
+
end
|
25
|
+
|
26
|
+
def get_game_summary_stats()
|
27
|
+
trophy_data = @agent.get(TROPHY_DATA_URL % [@username])
|
28
|
+
summary_stats = Hash.new
|
29
|
+
|
30
|
+
game_slots = trophy_data.search(".slot")
|
31
|
+
game_slots.each { |game_data|
|
32
|
+
game = Game.new
|
33
|
+
game.title = game_data.search(".titletext a").text.strip
|
34
|
+
game.uri = game_data.search(".titletext a").first[:href].strip
|
35
|
+
game.logo_uri = game_data.search(".titlelogo img").first[:src].strip
|
36
|
+
#TODO Make sure we got the right amount of trophy columns
|
37
|
+
trophies_earned = Hash.new
|
38
|
+
trophies_earned[:bronze] = game_data.search(".trophycount.normal .trophycontent")[0].text.strip
|
39
|
+
trophies_earned[:silver] = game_data.search(".trophycount.normal .trophycontent")[1].text.strip
|
40
|
+
trophies_earned[:gold] = game_data.search(".trophycount.normal .trophycontent")[2].text.strip
|
41
|
+
trophies_earned[:platinum] = game_data.search(".trophycount.normal .trophycontent")[3].text.strip
|
42
|
+
|
43
|
+
summary_stats[game] = trophies_earned
|
44
|
+
}
|
45
|
+
return summary_stats
|
46
|
+
end
|
47
|
+
|
48
|
+
def get_game_detail_stats(game_uri, trophy_levels)
|
49
|
+
trophy_info = Hash.new
|
50
|
+
hidden_trophy_counter = 1
|
51
|
+
|
52
|
+
titleId = game_uri.split("/")[-1]
|
53
|
+
trophy_data = @agent.post(TROPHY_DETAILED_DATA_URL % [@username], "titleId" => titleId, "sortBy" => "id_asc")
|
54
|
+
|
55
|
+
trophy_data.search(".slotcontent").each { |slot|
|
56
|
+
trophy = Trophy.new
|
57
|
+
trophy.name = slot.search(".trophyTitleSortField").text.strip
|
58
|
+
trophy.description = slot.search(".subtext").text.strip
|
59
|
+
|
60
|
+
# Game is a "spoiler trophy", ignore for now
|
61
|
+
if trophy.name == "" && trophy.description == ""
|
62
|
+
trophy.name = "Hidden Trophy %i" % hidden_trophy_counter
|
63
|
+
trophy.description = "Hidden Trophy"
|
64
|
+
hidden_trophy_counter += 1
|
65
|
+
end
|
66
|
+
|
67
|
+
trophy_level = slot.search(".trophyholder .trophyimage").first[:title]
|
68
|
+
trophy_levels.each do |level|
|
69
|
+
if trophy_level.downcase.include?(level.name.downcase)
|
70
|
+
level.trophies << trophy
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
unlock_date = slot.search(".lastTrophyTime").text.strip
|
75
|
+
|
76
|
+
if unlock_date.length > 0
|
77
|
+
trophy_info[trophy] = unlock_date
|
78
|
+
else
|
79
|
+
trophy_info[trophy] = nil
|
80
|
+
end
|
81
|
+
|
82
|
+
}
|
83
|
+
$logger.debug("Found trophies #{trophy_info}")
|
84
|
+
return trophy_info
|
85
|
+
end
|
86
|
+
|
87
|
+
protected
|
88
|
+
|
89
|
+
def request_account_trophy_page
|
90
|
+
@account_trophy_page = @agent.get(TROPHY_PAGE_URL % [@username])
|
91
|
+
end
|
92
|
+
|
93
|
+
private
|
94
|
+
|
95
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{trophy-scraper}
|
8
|
+
s.version = "1.0.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Vince Cima"]
|
12
|
+
s.date = %q{2009-10-15}
|
13
|
+
s.default_executable = %q{trophy-scraper}
|
14
|
+
s.description = %q{Collect and store trophy data for Playstation Network users.}
|
15
|
+
s.email = %q{contact@vincecima.com}
|
16
|
+
s.executables = ["trophy-scraper"]
|
17
|
+
s.extra_rdoc_files = [
|
18
|
+
"LICENSE",
|
19
|
+
"README.md"
|
20
|
+
]
|
21
|
+
s.files = [
|
22
|
+
".document",
|
23
|
+
".gitignore",
|
24
|
+
"LICENSE",
|
25
|
+
"README.md",
|
26
|
+
"Rakefile",
|
27
|
+
"SQL",
|
28
|
+
"TODO.md",
|
29
|
+
"VERSION",
|
30
|
+
"bin/trophy-scraper",
|
31
|
+
"lib/trophy_scraper.rb",
|
32
|
+
"lib/trophy_scraper/controller.rb",
|
33
|
+
"lib/trophy_scraper/core_ext.rb",
|
34
|
+
"lib/trophy_scraper/data_store.rb",
|
35
|
+
"lib/trophy_scraper/models/game.rb",
|
36
|
+
"lib/trophy_scraper/models/meta.rb",
|
37
|
+
"lib/trophy_scraper/models/trophy.rb",
|
38
|
+
"lib/trophy_scraper/models/trophy_level.rb",
|
39
|
+
"lib/trophy_scraper/models/unlock.rb",
|
40
|
+
"lib/trophy_scraper/models/user.rb",
|
41
|
+
"lib/trophy_scraper/models/user_stat.rb",
|
42
|
+
"lib/trophy_scraper/scraper.rb",
|
43
|
+
"trophy-scraper.gemspec"
|
44
|
+
]
|
45
|
+
s.homepage = %q{http://vincecima.com/blog/trophy-scraper/}
|
46
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
47
|
+
s.require_paths = ["lib"]
|
48
|
+
s.rubyforge_project = %q{trophy-scraper}
|
49
|
+
s.rubygems_version = %q{1.3.5}
|
50
|
+
s.summary = %q{Collect and store trophy data for Playstation Network users.}
|
51
|
+
|
52
|
+
if s.respond_to? :specification_version then
|
53
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
54
|
+
s.specification_version = 3
|
55
|
+
|
56
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
57
|
+
s.add_runtime_dependency(%q<dm-core>, [">= 0.9.11"])
|
58
|
+
s.add_runtime_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
59
|
+
s.add_runtime_dependency(%q<mechanize>, [">= 0.9.3"])
|
60
|
+
else
|
61
|
+
s.add_dependency(%q<dm-core>, [">= 0.9.11"])
|
62
|
+
s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
63
|
+
s.add_dependency(%q<mechanize>, [">= 0.9.3"])
|
64
|
+
end
|
65
|
+
else
|
66
|
+
s.add_dependency(%q<dm-core>, [">= 0.9.11"])
|
67
|
+
s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
68
|
+
s.add_dependency(%q<mechanize>, [">= 0.9.3"])
|
69
|
+
end
|
70
|
+
end
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: trophy-scraper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Vince Cima
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-10-15 00:00:00 -05:00
|
13
|
+
default_executable: trophy-scraper
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: dm-core
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: 0.9.11
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: do_sqlite3
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.9.12
|
34
|
+
version:
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: mechanize
|
37
|
+
type: :runtime
|
38
|
+
version_requirement:
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 0.9.3
|
44
|
+
version:
|
45
|
+
description: Collect and store trophy data for Playstation Network users.
|
46
|
+
email: contact@vincecima.com
|
47
|
+
executables:
|
48
|
+
- trophy-scraper
|
49
|
+
extensions: []
|
50
|
+
|
51
|
+
extra_rdoc_files:
|
52
|
+
- LICENSE
|
53
|
+
- README.md
|
54
|
+
files:
|
55
|
+
- .document
|
56
|
+
- .gitignore
|
57
|
+
- LICENSE
|
58
|
+
- README.md
|
59
|
+
- Rakefile
|
60
|
+
- SQL
|
61
|
+
- TODO.md
|
62
|
+
- VERSION
|
63
|
+
- bin/trophy-scraper
|
64
|
+
- lib/trophy_scraper.rb
|
65
|
+
- lib/trophy_scraper/controller.rb
|
66
|
+
- lib/trophy_scraper/core_ext.rb
|
67
|
+
- lib/trophy_scraper/data_store.rb
|
68
|
+
- lib/trophy_scraper/models/game.rb
|
69
|
+
- lib/trophy_scraper/models/meta.rb
|
70
|
+
- lib/trophy_scraper/models/trophy.rb
|
71
|
+
- lib/trophy_scraper/models/trophy_level.rb
|
72
|
+
- lib/trophy_scraper/models/unlock.rb
|
73
|
+
- lib/trophy_scraper/models/user.rb
|
74
|
+
- lib/trophy_scraper/models/user_stat.rb
|
75
|
+
- lib/trophy_scraper/scraper.rb
|
76
|
+
- trophy-scraper.gemspec
|
77
|
+
has_rdoc: true
|
78
|
+
homepage: http://vincecima.com/blog/trophy-scraper/
|
79
|
+
licenses: []
|
80
|
+
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options:
|
83
|
+
- --charset=UTF-8
|
84
|
+
require_paths:
|
85
|
+
- lib
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: "0"
|
91
|
+
version:
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: "0"
|
97
|
+
version:
|
98
|
+
requirements: []
|
99
|
+
|
100
|
+
rubyforge_project: trophy-scraper
|
101
|
+
rubygems_version: 1.3.5
|
102
|
+
signing_key:
|
103
|
+
specification_version: 3
|
104
|
+
summary: Collect and store trophy data for Playstation Network users.
|
105
|
+
test_files: []
|
106
|
+
|