trophy-scraper 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +6 -0
- data/LICENSE +21 -0
- data/README.md +39 -0
- data/Rakefile +64 -0
- data/SQL +6 -0
- data/TODO.md +10 -0
- data/VERSION +1 -0
- data/bin/trophy-scraper +94 -0
- data/lib/trophy_scraper.rb +68 -0
- data/lib/trophy_scraper/controller.rb +91 -0
- data/lib/trophy_scraper/core_ext.rb +22 -0
- data/lib/trophy_scraper/data_store.rb +40 -0
- data/lib/trophy_scraper/models/game.rb +11 -0
- data/lib/trophy_scraper/models/meta.rb +6 -0
- data/lib/trophy_scraper/models/trophy.rb +19 -0
- data/lib/trophy_scraper/models/trophy_level.rb +7 -0
- data/lib/trophy_scraper/models/unlock.rb +8 -0
- data/lib/trophy_scraper/models/user.rb +10 -0
- data/lib/trophy_scraper/models/user_stat.rb +21 -0
- data/lib/trophy_scraper/scraper.rb +95 -0
- data/trophy-scraper.gemspec +70 -0
- metadata +106 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2009 Vince Cima
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
## TrophyScraper
|
|
2
|
+
###Collect and store trophy data for Playstation Network users.
|
|
3
|
+
|
|
4
|
+
## Dependencies
|
|
5
|
+
* Sqlite
|
|
6
|
+
* DataMapper
|
|
7
|
+
* Mechanize
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
### Generic
|
|
11
|
+
gem install trophy-scraper
|
|
12
|
+
### Ubuntu/Debian
|
|
13
|
+
sudo aptitude install ruby ruby-dev rubygems sqlite3 libsqlite3-dev make libopenssl-ruby libxslt-dev libxml2-dev
|
|
14
|
+
sudo gem install trophy-scraper
|
|
15
|
+
The rubygems wrapped exectuable will be located at:
|
|
16
|
+
|
|
17
|
+
/var/lib/gems/1.8/bin/trophy-scraper
|
|
18
|
+
Make trophy-scraper available in the default PATH:
|
|
19
|
+
|
|
20
|
+
sudo ln -s /var/lib/gems/1.8/bin/trophy-scraper /usr/bin/trophy-scraper
|
|
21
|
+
|
|
22
|
+
## Usage
|
|
23
|
+
### Adding a user / Update a single user
|
|
24
|
+
trophy-scraper --username "Insert Username Here"
|
|
25
|
+
### Updating all users
|
|
26
|
+
trophy-scraper
|
|
27
|
+
### Specifying a custom database location
|
|
28
|
+
trophy-scraper --db "sqlite3://home/joe/trophy_scraper.db"
|
|
29
|
+
### Changing the logging options
|
|
30
|
+
trophy-scraper --log_level debug --log_dest "~/trophy-scraper.log" --db_log_level error --db_log_dest "~/trophy-scraper-db.log"
|
|
31
|
+
### Help
|
|
32
|
+
trophy-scraper --help
|
|
33
|
+
|
|
34
|
+
## Limitations
|
|
35
|
+
1. Some trophies are marked as "spoilers" and will never show their name and description.
|
|
36
|
+
1. Some games simply don't show up in the list of games played. Their trophies will contribute to a player's total however.
|
|
37
|
+
|
|
38
|
+
## Thanks
|
|
39
|
+
Thanks to the Haml and Jekyll projects for a little bit of code and lot of great examples on how to layout a Ruby project.
|
data/Rakefile
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'rake'
|
|
3
|
+
|
|
4
|
+
begin
|
|
5
|
+
require 'jeweler'
|
|
6
|
+
Jeweler::Tasks.new do |gem|
|
|
7
|
+
gem.name = "trophy-scraper"
|
|
8
|
+
gem.summary = %Q{Collect and store trophy data for Playstation Network users.}
|
|
9
|
+
gem.description = %Q{Collect and store trophy data for Playstation Network users.}
|
|
10
|
+
gem.email = "contact@vincecima.com"
|
|
11
|
+
gem.homepage = "http://vincecima.com/blog/trophy-scraper/"
|
|
12
|
+
gem.authors = ["Vince Cima"]
|
|
13
|
+
gem.rubyforge_project = "trophy-scraper"
|
|
14
|
+
gem.add_dependency("dm-core",">= 0.9.11")
|
|
15
|
+
gem.add_dependency("do_sqlite3",">= 0.9.12")
|
|
16
|
+
gem.add_dependency("mechanize",">= 0.9.3")
|
|
17
|
+
gem.rubyforge_project = "trophy-scraper"
|
|
18
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
|
19
|
+
end
|
|
20
|
+
Jeweler::GemcutterTasks.new
|
|
21
|
+
Jeweler::RubyforgeTasks.new do |rubyforge|
|
|
22
|
+
rubyforge.doc_task = "rdoc"
|
|
23
|
+
end
|
|
24
|
+
rescue LoadError
|
|
25
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
require 'rake/testtask'
|
|
29
|
+
Rake::TestTask.new(:test) do |test|
|
|
30
|
+
test.libs << 'lib' << 'test'
|
|
31
|
+
test.pattern = 'test/**/*_test.rb'
|
|
32
|
+
test.verbose = true
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
begin
|
|
36
|
+
require 'rcov/rcovtask'
|
|
37
|
+
Rcov::RcovTask.new do |test|
|
|
38
|
+
test.libs << 'test'
|
|
39
|
+
test.pattern = 'test/**/*_test.rb'
|
|
40
|
+
test.verbose = true
|
|
41
|
+
end
|
|
42
|
+
rescue LoadError
|
|
43
|
+
task :rcov do
|
|
44
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
task :test => :check_dependencies
|
|
49
|
+
|
|
50
|
+
task :default => :test
|
|
51
|
+
|
|
52
|
+
require 'rake/rdoctask'
|
|
53
|
+
Rake::RDocTask.new do |rdoc|
|
|
54
|
+
if File.exist?('VERSION')
|
|
55
|
+
version = File.read('VERSION')
|
|
56
|
+
else
|
|
57
|
+
version = ""
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
rdoc.rdoc_dir = 'rdoc'
|
|
61
|
+
rdoc.title = "trophy-scraper #{version}"
|
|
62
|
+
rdoc.rdoc_files.include('README*')
|
|
63
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
|
64
|
+
end
|
data/SQL
ADDED
data/TODO.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
## User Management
|
|
2
|
+
1. Add the ability to remove a user from the data store.
|
|
3
|
+
1. Add/Remove groups of users without triggering an update.
|
|
4
|
+
|
|
5
|
+
## Trophy Handling
|
|
6
|
+
1. Detect trophies that are displayed on Playstation.com and log out message.
|
|
7
|
+
1. Better handle "spoiler" trophies.
|
|
8
|
+
|
|
9
|
+
## Scraper
|
|
10
|
+
1. Add detection of page structure changes.
|
data/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.0.0
|
data/bin/trophy-scraper
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
|
|
4
|
+
|
|
5
|
+
help = <<HELP
|
|
6
|
+
TrophyScraper downloads trophy information for Playstation Network Accounts.
|
|
7
|
+
|
|
8
|
+
Basic Command Line Usage:
|
|
9
|
+
trophy-scraper # Update all users in database.
|
|
10
|
+
trophy-scraper --username "username" # Update "username".
|
|
11
|
+
trophy-scraper --db "sqlite3::memory:" # DataMapper db string.
|
|
12
|
+
|
|
13
|
+
Configuration is read from '~/.trophy_scraper.yml' but can be overriden
|
|
14
|
+
using the following options:
|
|
15
|
+
|
|
16
|
+
HELP
|
|
17
|
+
|
|
18
|
+
require 'optparse'
|
|
19
|
+
require 'trophy_scraper'
|
|
20
|
+
|
|
21
|
+
options = {}
|
|
22
|
+
opts = OptionParser.new do |opts|
|
|
23
|
+
opts.banner = help
|
|
24
|
+
|
|
25
|
+
opts.on("--config [PATH]", "Full path to yml configuration file.") do |path|
|
|
26
|
+
options['config'] = path
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
opts.on("--db [DB_STRING]", "DataMapper database string.") do |db_string|
|
|
30
|
+
options['db'] = db_string
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
opts.on("--username [USERNAME]", "Username to run the trophy update for.") do |username|
|
|
34
|
+
options['username'] = username
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
opts.on("--log-level [LOG_LEVEL]", "Standard Ruby logger output level.") do |log_level|
|
|
38
|
+
if log_level.downcase == "debug"
|
|
39
|
+
options['log_level'] = Logger::DEBUG
|
|
40
|
+
elsif log_level.downcase == "info"
|
|
41
|
+
options['log_level'] = Logger::INFO
|
|
42
|
+
elsif log_level.downcase == "warn"
|
|
43
|
+
options['log_level'] = Logger::WARN
|
|
44
|
+
elsif log_level.downcase == "error"
|
|
45
|
+
options['log_level'] = Logger::ERROR
|
|
46
|
+
elsif log_level.downcase == "fatal"
|
|
47
|
+
options['log_level'] = Logger::FATAL
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
opts.on("--log-dest [LOG_DEST]", "STDOUT, STDERR or full path to log file.") do |log_dest|
|
|
52
|
+
if log_dest.downcase == "STDOUT".downcase
|
|
53
|
+
options['log_dest'] = STDOUT
|
|
54
|
+
elsif log_dest.downcase == "STDERR".downcase
|
|
55
|
+
options['log_dest'] = STDERR
|
|
56
|
+
else
|
|
57
|
+
options['log_dest'] = File.expand_path(log_dest)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
opts.on("--db-log-level [LOG_LEVEL]", "Ruby logger output level for database.") do |log_level|
|
|
62
|
+
if log_level.downcase == "off"
|
|
63
|
+
options['db_log_level'] = :off
|
|
64
|
+
elsif log_level.downcase == "debug"
|
|
65
|
+
options['db_log_level'] = :debug
|
|
66
|
+
elsif log_level.downcase == "info"
|
|
67
|
+
options['db_log_level'] = :info
|
|
68
|
+
elsif log_level.downcase == "warn"
|
|
69
|
+
options['db_log_level'] = :warn
|
|
70
|
+
elsif log_level.downcase == "error"
|
|
71
|
+
options['db_log_level'] = :error
|
|
72
|
+
elsif log_level.downcase == "fatal"
|
|
73
|
+
options['db_log_level'] = :fatal
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
opts.on("--db-log-dest [LOG_DEST]", "STDOUT, STDERR or full path to db log file.") do |db_log_dest|
|
|
78
|
+
if db_log_dest.downcase == "STDOUT".downcase
|
|
79
|
+
options['db_log_dest'] = STDOUT
|
|
80
|
+
elsif db_log_dest.downcase == "STDERR".downcase
|
|
81
|
+
options['db_log_dest'] = STDERR
|
|
82
|
+
else
|
|
83
|
+
options['db_log_dest'] = File.expand_path(db_log_dest)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Read command line options into `options` hash
|
|
89
|
+
opts.parse!
|
|
90
|
+
|
|
91
|
+
options = TrophyScraper.configuration(options)
|
|
92
|
+
|
|
93
|
+
controller = Controller.new(options)
|
|
94
|
+
controller.process
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
$:.unshift File.dirname(__FILE__) # For use/testing when no gem is installed
|
|
2
|
+
|
|
3
|
+
# core
|
|
4
|
+
|
|
5
|
+
# stdlib
|
|
6
|
+
require "logger"
|
|
7
|
+
|
|
8
|
+
# 3rd party
|
|
9
|
+
require 'dm-core'
|
|
10
|
+
require 'mechanize'
|
|
11
|
+
|
|
12
|
+
# internal requires
|
|
13
|
+
require "trophy_scraper/controller"
|
|
14
|
+
require "trophy_scraper/core_ext"
|
|
15
|
+
require "trophy_scraper/data_store"
|
|
16
|
+
require "trophy_scraper/scraper"
|
|
17
|
+
require "trophy_scraper/models/meta"
|
|
18
|
+
require "trophy_scraper/models/user"
|
|
19
|
+
require "trophy_scraper/models/user_stat"
|
|
20
|
+
require "trophy_scraper/models/unlock"
|
|
21
|
+
require "trophy_scraper/models/trophy"
|
|
22
|
+
require "trophy_scraper/models/game"
|
|
23
|
+
require "trophy_scraper/models/trophy_level"
|
|
24
|
+
|
|
25
|
+
module TrophyScraper
|
|
26
|
+
# Default options. Overriden by values in _config.yml or command-line opts.
|
|
27
|
+
# (Strings rather symbols used for compatability with YAML)
|
|
28
|
+
DEFAULTS = {
|
|
29
|
+
'config' => '~/.trophy_scraper.yml',
|
|
30
|
+
'db' => '',
|
|
31
|
+
'log_level' => Logger::INFO,
|
|
32
|
+
'log_dest' => STDOUT,
|
|
33
|
+
'db' => "sqlite3://%s/trophy_scraper.db" % File.expand_path("~"),
|
|
34
|
+
'db_log_level' => "fatal",
|
|
35
|
+
'db_log_dest' => STDOUT
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
# Generate a TrophyScraper configuration Hash by merging the default options
|
|
39
|
+
# with anything in config.yml, and adding the given options on top
|
|
40
|
+
# +override+ is a Hash of config directives
|
|
41
|
+
#
|
|
42
|
+
# Returns Hash
|
|
43
|
+
def self.configuration(override)
|
|
44
|
+
# config.yml may override default source location, but until
|
|
45
|
+
# then, we need to know where to look for config.yml
|
|
46
|
+
source = override['config'] || TrophyScraper::DEFAULTS['config']
|
|
47
|
+
|
|
48
|
+
# Get configuration from <source>
|
|
49
|
+
config_file = File.expand_path(source)
|
|
50
|
+
begin
|
|
51
|
+
config = YAML.load_file(config_file)
|
|
52
|
+
raise "Invalid configuration - #{config_file}" if !config.is_a?(Hash)
|
|
53
|
+
STDOUT.puts "Configuration from #{config_file}"
|
|
54
|
+
rescue => err
|
|
55
|
+
STDERR.puts "WARNING: Could not read configuration. Using defaults (and options)."
|
|
56
|
+
STDERR.puts "\t" + err.to_s
|
|
57
|
+
config = {}
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Merge DEFAULTS < config.yml < override
|
|
61
|
+
TrophyScraper::DEFAULTS.deep_merge(config).deep_merge(override)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.version
|
|
65
|
+
yml = YAML.load(File.read(File.join(File.dirname(__FILE__), *%w[.. VERSION.yml])))
|
|
66
|
+
"#{yml[:major]}.#{yml[:minor]}.#{yml[:patch]}"
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
class Controller
|
|
2
|
+
def initialize(options)
|
|
3
|
+
@options = options.clone
|
|
4
|
+
$logger = Logger.new(@options['log_dest'])
|
|
5
|
+
$logger.level = @options['log_level']
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
def process
|
|
9
|
+
#Initialize connection to db and do any version migration needed.
|
|
10
|
+
data_store = DataStore.new(@options['db'], @options['db_log_level'], @options['db_log_dest'])
|
|
11
|
+
|
|
12
|
+
users_to_check = get_users_to_check
|
|
13
|
+
trophy_levels = TrophyLevel.all
|
|
14
|
+
|
|
15
|
+
users_to_check.each do |user|
|
|
16
|
+
$logger.info("Preparing to update #{user.username}.")
|
|
17
|
+
scraper = Scraper.new(user.username)
|
|
18
|
+
|
|
19
|
+
user_summary_stats = scraper.get_user_summary_stats
|
|
20
|
+
if user_summary_stats.total_trophies > user.unlocks.size
|
|
21
|
+
$logger.info("Website reports more trophies for #{user.username} than data store.")
|
|
22
|
+
user.user_stats << user_summary_stats
|
|
23
|
+
|
|
24
|
+
games_summary_stats = scraper.get_game_summary_stats()
|
|
25
|
+
games_summary_stats.each_pair do |found_game, trophies_earned|
|
|
26
|
+
game = Game.first(:title => found_game.title)
|
|
27
|
+
|
|
28
|
+
if game == nil
|
|
29
|
+
game = found_game
|
|
30
|
+
|
|
31
|
+
game.save
|
|
32
|
+
$logger.debug("Saved new game: #{game.title}")
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
trophy_info = scraper.get_game_detail_stats(found_game.uri, trophy_levels)
|
|
36
|
+
trophy_info.each_pair do |found_trophy, earned_date|
|
|
37
|
+
trophy = game.trophies.first(:name => found_trophy.name, :game_id => game.id, :trophy_level_id => found_trophy.trophy_level_id)
|
|
38
|
+
if trophy == nil
|
|
39
|
+
trophy = found_trophy
|
|
40
|
+
|
|
41
|
+
game.trophies << trophy
|
|
42
|
+
trophy.save
|
|
43
|
+
$logger.debug("Saved new trophy: #{trophy.name}")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
if earned_date != nil
|
|
47
|
+
unlock = Unlock.first(:user_username => user.username, :trophy_id => trophy.id)
|
|
48
|
+
if unlock == nil
|
|
49
|
+
unlock = Unlock.new(:earned_on => earned_date)
|
|
50
|
+
trophy.unlocks << unlock
|
|
51
|
+
user.unlocks << unlock
|
|
52
|
+
unlock.save
|
|
53
|
+
$logger.info("#{user.username} has unlocked '#{trophy.name}' in '#{game.title}'.")
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
elsif user_summary_stats.total_trophies == user.unlocks.size
|
|
59
|
+
$logger.info("Same number of trophies on website and in data store. Nothing to update for #{user.username}")
|
|
60
|
+
else
|
|
61
|
+
$logger.warn("Website reports less trophies than data store for #{user.username}. Something is wrong.")
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
user.last_updated = Time.now
|
|
65
|
+
user.save
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
$logger.close
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def get_users_to_check
|
|
74
|
+
users_to_check = Array.new
|
|
75
|
+
if @options['username'] != nil
|
|
76
|
+
user = User.get(@options['username'])
|
|
77
|
+
if user == nil
|
|
78
|
+
user = User.new(:username => @options['username'])
|
|
79
|
+
user.save
|
|
80
|
+
$logger.info("#{user.username} is a new user.")
|
|
81
|
+
end
|
|
82
|
+
users_to_check.push(user)
|
|
83
|
+
$logger.info("#{user.username} will be the only user checked.")
|
|
84
|
+
else
|
|
85
|
+
users_to_check = User.all
|
|
86
|
+
$logger.info("All users in data store will be checked.")
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return users_to_check
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class Hash
|
|
2
|
+
# Merges self with another hash, recursively.
|
|
3
|
+
#
|
|
4
|
+
# This code was lovingly stolen from some random gem:
|
|
5
|
+
# http://gemjack.com/gems/tartan-0.1.1/classes/Hash.html
|
|
6
|
+
#
|
|
7
|
+
# Thanks to whoever made it.
|
|
8
|
+
def deep_merge(hash)
|
|
9
|
+
target = dup
|
|
10
|
+
|
|
11
|
+
hash.keys.each do |key|
|
|
12
|
+
if hash[key].is_a? Hash and self[key].is_a? Hash
|
|
13
|
+
target[key] = target[key].deep_merge(hash[key])
|
|
14
|
+
next
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
target[key] = hash[key]
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
target
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
class DataStore
|
|
2
|
+
@@DB_VERSION = 1
|
|
3
|
+
def initialize(connection_string, logger_level, logger_dest)
|
|
4
|
+
DataMapper::Logger.new(logger_dest, logger_level)
|
|
5
|
+
DataMapper.setup(:default, connection_string)
|
|
6
|
+
|
|
7
|
+
begin
|
|
8
|
+
version_meta = Meta.get("version")
|
|
9
|
+
rescue
|
|
10
|
+
#TODO, only eat the table not found error
|
|
11
|
+
end
|
|
12
|
+
if version_meta == nil
|
|
13
|
+
$logger.info("Database is empty, creating schema.")
|
|
14
|
+
DataMapper.auto_migrate!
|
|
15
|
+
version_meta = Meta.new(:name => "version", :value => @@DB_VERSION)
|
|
16
|
+
version_meta.save
|
|
17
|
+
insert_reference_data
|
|
18
|
+
elsif version_meta.value.to_i < @@DB_VERSION
|
|
19
|
+
$logger.info("Database schema version is older than code's, gracefully updating.")
|
|
20
|
+
DataMapper.auto_upgrade!
|
|
21
|
+
version_meta.value = @@DB_VERSION
|
|
22
|
+
version_meta.save
|
|
23
|
+
elsif version_meta.value.to_i > @@DB_VERSION
|
|
24
|
+
#TODO Code is older than db, warn and abort.
|
|
25
|
+
$logger.warn("Database schema version is newer than code's, aborting.")
|
|
26
|
+
else
|
|
27
|
+
$logger.info("Database schema is up to date.")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def insert_reference_data
|
|
34
|
+
TrophyLevel.new(:name => "Platinum").save
|
|
35
|
+
TrophyLevel.new(:name => "Gold").save
|
|
36
|
+
TrophyLevel.new(:name => "Silver").save
|
|
37
|
+
TrophyLevel.new(:name => "Bronze").save
|
|
38
|
+
TrophyLevel.new(:name => "Hidden").save
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
class Trophy
|
|
2
|
+
include DataMapper::Resource
|
|
3
|
+
|
|
4
|
+
property :id, Serial
|
|
5
|
+
property :name, String, :nullable => false
|
|
6
|
+
property :description, String, :nullable => false
|
|
7
|
+
property :icon, Object
|
|
8
|
+
|
|
9
|
+
has n, :unlocks
|
|
10
|
+
has n, :users, :through => :unlocks
|
|
11
|
+
belongs_to :game
|
|
12
|
+
belongs_to :trophy_level
|
|
13
|
+
|
|
14
|
+
attr_accessor :icon_uri
|
|
15
|
+
|
|
16
|
+
def to_s
|
|
17
|
+
return "id = #{@id}, name = #{@name}, description = #{@description}"
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
class UserStat
|
|
2
|
+
include DataMapper::Resource
|
|
3
|
+
|
|
4
|
+
property :id, Serial
|
|
5
|
+
property :number_of_bronze, Integer
|
|
6
|
+
property :number_of_silver, Integer
|
|
7
|
+
property :number_of_gold, Integer
|
|
8
|
+
property :number_of_platinum, Integer
|
|
9
|
+
property :level, Integer
|
|
10
|
+
property :percent_to_next_level, Integer
|
|
11
|
+
|
|
12
|
+
belongs_to :user
|
|
13
|
+
|
|
14
|
+
def total_trophies
|
|
15
|
+
return @number_of_bronze + @number_of_silver + @number_of_gold + @number_of_platinum
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def to_s
|
|
19
|
+
return "percent_to_next_level = #{@percent_to_next_level}, level = #{@level}, #bronze = #{@number_of_bronze}, #silver = #{@number_of_silver}, #gold = #{@number_of_gold}, #platinum = #{@number_of_platinum}"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
class Scraper
|
|
2
|
+
TROPHY_PAGE_URL = "http://profiles.us.playstation.com/playstation/psn/profiles/%s"
|
|
3
|
+
TROPHY_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_trophies_data"
|
|
4
|
+
TROPHY_DETAILED_DATA_URL = "http://profiles.us.playstation.com/playstation/psn/profile/%s/get_ordered_title_details_data"
|
|
5
|
+
|
|
6
|
+
def initialize(username)
|
|
7
|
+
@username = username
|
|
8
|
+
@agent = WWW::Mechanize.new
|
|
9
|
+
@update_date = Time.now
|
|
10
|
+
@account_trophy_page = nil
|
|
11
|
+
self.request_account_trophy_page
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def get_user_summary_stats
|
|
15
|
+
stats = UserStat.new
|
|
16
|
+
stats.level = @account_trophy_page.search("#leveltext").text.strip
|
|
17
|
+
stats.percent_to_next_level = @account_trophy_page.search(".progresstext").text.strip.delete("%")
|
|
18
|
+
stats.number_of_bronze = @account_trophy_page.search(".podium .bronze").text.strip.split[0]
|
|
19
|
+
stats.number_of_silver = @account_trophy_page.search(".podium .silver").text.strip.split[0]
|
|
20
|
+
stats.number_of_gold = @account_trophy_page.search(".podium .gold").text.strip.split[0]
|
|
21
|
+
stats.number_of_platinum = @account_trophy_page.search(".podium .platinum").text.strip.split[0]
|
|
22
|
+
$logger.debug("Found user stats #{stats}")
|
|
23
|
+
return stats
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def get_game_summary_stats()
|
|
27
|
+
trophy_data = @agent.get(TROPHY_DATA_URL % [@username])
|
|
28
|
+
summary_stats = Hash.new
|
|
29
|
+
|
|
30
|
+
game_slots = trophy_data.search(".slot")
|
|
31
|
+
game_slots.each { |game_data|
|
|
32
|
+
game = Game.new
|
|
33
|
+
game.title = game_data.search(".titletext a").text.strip
|
|
34
|
+
game.uri = game_data.search(".titletext a").first[:href].strip
|
|
35
|
+
game.logo_uri = game_data.search(".titlelogo img").first[:src].strip
|
|
36
|
+
#TODO Make sure we got the right amount of trophy columns
|
|
37
|
+
trophies_earned = Hash.new
|
|
38
|
+
trophies_earned[:bronze] = game_data.search(".trophycount.normal .trophycontent")[0].text.strip
|
|
39
|
+
trophies_earned[:silver] = game_data.search(".trophycount.normal .trophycontent")[1].text.strip
|
|
40
|
+
trophies_earned[:gold] = game_data.search(".trophycount.normal .trophycontent")[2].text.strip
|
|
41
|
+
trophies_earned[:platinum] = game_data.search(".trophycount.normal .trophycontent")[3].text.strip
|
|
42
|
+
|
|
43
|
+
summary_stats[game] = trophies_earned
|
|
44
|
+
}
|
|
45
|
+
return summary_stats
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def get_game_detail_stats(game_uri, trophy_levels)
|
|
49
|
+
trophy_info = Hash.new
|
|
50
|
+
hidden_trophy_counter = 1
|
|
51
|
+
|
|
52
|
+
titleId = game_uri.split("/")[-1]
|
|
53
|
+
trophy_data = @agent.post(TROPHY_DETAILED_DATA_URL % [@username], "titleId" => titleId, "sortBy" => "id_asc")
|
|
54
|
+
|
|
55
|
+
trophy_data.search(".slotcontent").each { |slot|
|
|
56
|
+
trophy = Trophy.new
|
|
57
|
+
trophy.name = slot.search(".trophyTitleSortField").text.strip
|
|
58
|
+
trophy.description = slot.search(".subtext").text.strip
|
|
59
|
+
|
|
60
|
+
# Game is a "spoiler trophy", ignore for now
|
|
61
|
+
if trophy.name == "" && trophy.description == ""
|
|
62
|
+
trophy.name = "Hidden Trophy %i" % hidden_trophy_counter
|
|
63
|
+
trophy.description = "Hidden Trophy"
|
|
64
|
+
hidden_trophy_counter += 1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
trophy_level = slot.search(".trophyholder .trophyimage").first[:title]
|
|
68
|
+
trophy_levels.each do |level|
|
|
69
|
+
if trophy_level.downcase.include?(level.name.downcase)
|
|
70
|
+
level.trophies << trophy
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
unlock_date = slot.search(".lastTrophyTime").text.strip
|
|
75
|
+
|
|
76
|
+
if unlock_date.length > 0
|
|
77
|
+
trophy_info[trophy] = unlock_date
|
|
78
|
+
else
|
|
79
|
+
trophy_info[trophy] = nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
}
|
|
83
|
+
$logger.debug("Found trophies #{trophy_info}")
|
|
84
|
+
return trophy_info
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
protected
|
|
88
|
+
|
|
89
|
+
def request_account_trophy_page
|
|
90
|
+
@account_trophy_page = @agent.get(TROPHY_PAGE_URL % [@username])
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Generated by jeweler
|
|
2
|
+
# DO NOT EDIT THIS FILE
|
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
|
4
|
+
# -*- encoding: utf-8 -*-
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |s|
|
|
7
|
+
s.name = %q{trophy-scraper}
|
|
8
|
+
s.version = "1.0.0"
|
|
9
|
+
|
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
|
+
s.authors = ["Vince Cima"]
|
|
12
|
+
s.date = %q{2009-10-15}
|
|
13
|
+
s.default_executable = %q{trophy-scraper}
|
|
14
|
+
s.description = %q{Collect and store trophy data for Playstation Network users.}
|
|
15
|
+
s.email = %q{contact@vincecima.com}
|
|
16
|
+
s.executables = ["trophy-scraper"]
|
|
17
|
+
s.extra_rdoc_files = [
|
|
18
|
+
"LICENSE",
|
|
19
|
+
"README.md"
|
|
20
|
+
]
|
|
21
|
+
s.files = [
|
|
22
|
+
".document",
|
|
23
|
+
".gitignore",
|
|
24
|
+
"LICENSE",
|
|
25
|
+
"README.md",
|
|
26
|
+
"Rakefile",
|
|
27
|
+
"SQL",
|
|
28
|
+
"TODO.md",
|
|
29
|
+
"VERSION",
|
|
30
|
+
"bin/trophy-scraper",
|
|
31
|
+
"lib/trophy_scraper.rb",
|
|
32
|
+
"lib/trophy_scraper/controller.rb",
|
|
33
|
+
"lib/trophy_scraper/core_ext.rb",
|
|
34
|
+
"lib/trophy_scraper/data_store.rb",
|
|
35
|
+
"lib/trophy_scraper/models/game.rb",
|
|
36
|
+
"lib/trophy_scraper/models/meta.rb",
|
|
37
|
+
"lib/trophy_scraper/models/trophy.rb",
|
|
38
|
+
"lib/trophy_scraper/models/trophy_level.rb",
|
|
39
|
+
"lib/trophy_scraper/models/unlock.rb",
|
|
40
|
+
"lib/trophy_scraper/models/user.rb",
|
|
41
|
+
"lib/trophy_scraper/models/user_stat.rb",
|
|
42
|
+
"lib/trophy_scraper/scraper.rb",
|
|
43
|
+
"trophy-scraper.gemspec"
|
|
44
|
+
]
|
|
45
|
+
s.homepage = %q{http://vincecima.com/blog/trophy-scraper/}
|
|
46
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
|
47
|
+
s.require_paths = ["lib"]
|
|
48
|
+
s.rubyforge_project = %q{trophy-scraper}
|
|
49
|
+
s.rubygems_version = %q{1.3.5}
|
|
50
|
+
s.summary = %q{Collect and store trophy data for Playstation Network users.}
|
|
51
|
+
|
|
52
|
+
if s.respond_to? :specification_version then
|
|
53
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
|
54
|
+
s.specification_version = 3
|
|
55
|
+
|
|
56
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
|
57
|
+
s.add_runtime_dependency(%q<dm-core>, [">= 0.9.11"])
|
|
58
|
+
s.add_runtime_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
|
59
|
+
s.add_runtime_dependency(%q<mechanize>, [">= 0.9.3"])
|
|
60
|
+
else
|
|
61
|
+
s.add_dependency(%q<dm-core>, [">= 0.9.11"])
|
|
62
|
+
s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
|
63
|
+
s.add_dependency(%q<mechanize>, [">= 0.9.3"])
|
|
64
|
+
end
|
|
65
|
+
else
|
|
66
|
+
s.add_dependency(%q<dm-core>, [">= 0.9.11"])
|
|
67
|
+
s.add_dependency(%q<do_sqlite3>, [">= 0.9.12"])
|
|
68
|
+
s.add_dependency(%q<mechanize>, [">= 0.9.3"])
|
|
69
|
+
end
|
|
70
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: trophy-scraper
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 1.0.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Vince Cima
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
|
|
12
|
+
date: 2009-10-15 00:00:00 -05:00
|
|
13
|
+
default_executable: trophy-scraper
|
|
14
|
+
dependencies:
|
|
15
|
+
- !ruby/object:Gem::Dependency
|
|
16
|
+
name: dm-core
|
|
17
|
+
type: :runtime
|
|
18
|
+
version_requirement:
|
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
20
|
+
requirements:
|
|
21
|
+
- - ">="
|
|
22
|
+
- !ruby/object:Gem::Version
|
|
23
|
+
version: 0.9.11
|
|
24
|
+
version:
|
|
25
|
+
- !ruby/object:Gem::Dependency
|
|
26
|
+
name: do_sqlite3
|
|
27
|
+
type: :runtime
|
|
28
|
+
version_requirement:
|
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: 0.9.12
|
|
34
|
+
version:
|
|
35
|
+
- !ruby/object:Gem::Dependency
|
|
36
|
+
name: mechanize
|
|
37
|
+
type: :runtime
|
|
38
|
+
version_requirement:
|
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
40
|
+
requirements:
|
|
41
|
+
- - ">="
|
|
42
|
+
- !ruby/object:Gem::Version
|
|
43
|
+
version: 0.9.3
|
|
44
|
+
version:
|
|
45
|
+
description: Collect and store trophy data for Playstation Network users.
|
|
46
|
+
email: contact@vincecima.com
|
|
47
|
+
executables:
|
|
48
|
+
- trophy-scraper
|
|
49
|
+
extensions: []
|
|
50
|
+
|
|
51
|
+
extra_rdoc_files:
|
|
52
|
+
- LICENSE
|
|
53
|
+
- README.md
|
|
54
|
+
files:
|
|
55
|
+
- .document
|
|
56
|
+
- .gitignore
|
|
57
|
+
- LICENSE
|
|
58
|
+
- README.md
|
|
59
|
+
- Rakefile
|
|
60
|
+
- SQL
|
|
61
|
+
- TODO.md
|
|
62
|
+
- VERSION
|
|
63
|
+
- bin/trophy-scraper
|
|
64
|
+
- lib/trophy_scraper.rb
|
|
65
|
+
- lib/trophy_scraper/controller.rb
|
|
66
|
+
- lib/trophy_scraper/core_ext.rb
|
|
67
|
+
- lib/trophy_scraper/data_store.rb
|
|
68
|
+
- lib/trophy_scraper/models/game.rb
|
|
69
|
+
- lib/trophy_scraper/models/meta.rb
|
|
70
|
+
- lib/trophy_scraper/models/trophy.rb
|
|
71
|
+
- lib/trophy_scraper/models/trophy_level.rb
|
|
72
|
+
- lib/trophy_scraper/models/unlock.rb
|
|
73
|
+
- lib/trophy_scraper/models/user.rb
|
|
74
|
+
- lib/trophy_scraper/models/user_stat.rb
|
|
75
|
+
- lib/trophy_scraper/scraper.rb
|
|
76
|
+
- trophy-scraper.gemspec
|
|
77
|
+
has_rdoc: true
|
|
78
|
+
homepage: http://vincecima.com/blog/trophy-scraper/
|
|
79
|
+
licenses: []
|
|
80
|
+
|
|
81
|
+
post_install_message:
|
|
82
|
+
rdoc_options:
|
|
83
|
+
- --charset=UTF-8
|
|
84
|
+
require_paths:
|
|
85
|
+
- lib
|
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
87
|
+
requirements:
|
|
88
|
+
- - ">="
|
|
89
|
+
- !ruby/object:Gem::Version
|
|
90
|
+
version: "0"
|
|
91
|
+
version:
|
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: "0"
|
|
97
|
+
version:
|
|
98
|
+
requirements: []
|
|
99
|
+
|
|
100
|
+
rubyforge_project: trophy-scraper
|
|
101
|
+
rubygems_version: 1.3.5
|
|
102
|
+
signing_key:
|
|
103
|
+
specification_version: 3
|
|
104
|
+
summary: Collect and store trophy data for Playstation Network users.
|
|
105
|
+
test_files: []
|
|
106
|
+
|