hltv_scrapping 0.2.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.DS_Store +0 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +35 -0
- data/LICENSE.txt +21 -0
- data/README.md +82 -0
- data/Rakefile +9 -0
- data/bin/console +41 -0
- data/bin/setup +8 -0
- data/hltv_scrapping.gemspec +36 -0
- data/lib/.DS_Store +0 -0
- data/lib/hltv_scrapping.rb +269 -0
- data/lib/hltv_scrapping/version.rb +3 -0
- data/lib/tasks/generate_models.rake +29 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: df2409b3d37ed3b56945fd9aeb26c32fafaeeb30
|
4
|
+
data.tar.gz: fa0727154982821f76ef64b472475553b33d1b87
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 584be4e609176a130fd1d2dda85cbc0de117b8a4c9b682712458ca0eb40b6fe0511d858fc4b7286322a14e59b58e532d0249846b84fca97b01845b0ba3cd40e9
|
7
|
+
data.tar.gz: c8bd29d068d9260204a12f54bcd3a710dc8472f9454b82785b659bc20525523e72a637e9dc7f69a7160135948e4034bd2a700fd957ab8da607da6fae518cd743
|
data/.DS_Store
ADDED
Binary file
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
hltv_scrapping (0.2.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
diff-lcs (1.3)
|
10
|
+
rake (10.5.0)
|
11
|
+
rspec (3.6.0)
|
12
|
+
rspec-core (~> 3.6.0)
|
13
|
+
rspec-expectations (~> 3.6.0)
|
14
|
+
rspec-mocks (~> 3.6.0)
|
15
|
+
rspec-core (3.6.0)
|
16
|
+
rspec-support (~> 3.6.0)
|
17
|
+
rspec-expectations (3.6.0)
|
18
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
19
|
+
rspec-support (~> 3.6.0)
|
20
|
+
rspec-mocks (3.6.0)
|
21
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
22
|
+
rspec-support (~> 3.6.0)
|
23
|
+
rspec-support (3.6.0)
|
24
|
+
|
25
|
+
PLATFORMS
|
26
|
+
ruby
|
27
|
+
|
28
|
+
DEPENDENCIES
|
29
|
+
bundler (~> 1.14)
|
30
|
+
hltv_scrapping!
|
31
|
+
rake (~> 10.0)
|
32
|
+
rspec (~> 3.0)
|
33
|
+
|
34
|
+
BUNDLED WITH
|
35
|
+
1.14.6
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 ronan louarn
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
# HltvScrapping
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/hltv_scrapping`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'hltv_scrapping'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install hltv_scrapping
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
On a new project:
|
26
|
+
|
27
|
+
First step => models generation:
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
HltvScrapping::Call.generate_models
|
31
|
+
```
|
32
|
+
|
33
|
+
Second step => Scrap all team on hltv:
|
34
|
+
|
35
|
+
```ruby
|
36
|
+
HltvScrapping::Call.get_all_teams
|
37
|
+
```
|
38
|
+
|
39
|
+
Third step => Scrap all matches on hltv and they will be associated with teams, then you can use 'Team.first.matches':
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
HltvScrapping::Call.get_all_matches
|
43
|
+
```
|
44
|
+
|
45
|
+
Fourth step => Scrap all stats/map on hltv and they will be associated with teams, then you can use 'Team.first.map_stats':
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
HltvScrapping::Call.update_stats
|
49
|
+
```
|
50
|
+
|
51
|
+
Fifth step => Scrap all players on hltv and they will be associated with teams, then you can use 'Team.first.players':
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
HltvScrapping::Call.get_players
|
55
|
+
```
|
56
|
+
|
57
|
+
Sixth step => Scrap all players global stats on hltv and they will be associated with teams, then you can use 'Player.first.global_stats':
|
58
|
+
|
59
|
+
```ruby
|
60
|
+
HltvScrapping::Call.get_players_global_stats
|
61
|
+
```
|
62
|
+
Seventh step => Scrap all players match_stats on hltv and they will be associated with teams, then you can use 'Player.first.player_participation_matches':
|
63
|
+
|
64
|
+
```ruby
|
65
|
+
HltvScrapping::Call.get_player_participation
|
66
|
+
```
|
67
|
+
All done! You have all hltv csgo stats.
|
68
|
+
## Development
|
69
|
+
|
70
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
71
|
+
|
72
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
73
|
+
|
74
|
+
## Contributing
|
75
|
+
|
76
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/hltv_scrapping.
|
77
|
+
|
78
|
+
|
79
|
+
## License
|
80
|
+
|
81
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
82
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "hltv_scrapping"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
15
|
+
|
16
|
+
puts "Génération des team"
|
17
|
+
# team generator
|
18
|
+
system("rails g model team name maps:integer kd_diff:integer kd:float rating:float link")
|
19
|
+
|
20
|
+
puts "Généation des matchs"
|
21
|
+
# matches generator
|
22
|
+
system("rails g model match event date:datetime map key link")
|
23
|
+
|
24
|
+
puts "Génération des map stats"
|
25
|
+
# team map stats generator
|
26
|
+
system("rails g model map_stat map team:references win draw losses win_rate:integer total_rounds:integer round_win_after_first_kill_percent:integer round_win_after_first_death_percent:integer")
|
27
|
+
|
28
|
+
puts "Génération de player"
|
29
|
+
# players generator
|
30
|
+
system("rails g model player link team:references maps_played:integer name actif:boolean")
|
31
|
+
|
32
|
+
puts "Génération de player global stat"
|
33
|
+
#player globals stats generator
|
34
|
+
system("rails g model player_global_stat total_kills:integer headshot_percent:float total_deaths:integer kd:float damage_per_round:float grenade_dmg_per_round:float maps_played:integer rounds_played:integer kills_per_round:float assist_per_round:float death_per_round:float saved_by_teammates_per_round:float saved_teammates_per_round:float player:references")
|
35
|
+
|
36
|
+
puts "Génération de participation de joueurs"
|
37
|
+
#Play participation match generator
|
38
|
+
system("rails g model player_participation_match player:references kills:integer head_shots:integer assists:integer deaths:integer kast_percent:float kd_diff:integer average_damage_per_rounds:float fk_diff:integer participation:references")
|
39
|
+
|
40
|
+
#migrate migrations
|
41
|
+
system("rails db:migrate")
|
data/bin/setup
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'hltv_scrapping/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "hltv_scrapping"
|
8
|
+
spec.version = HltvScrapping::VERSION
|
9
|
+
spec.authors = ["ronan louarn"]
|
10
|
+
spec.email = ["ronan33720@hotmail.Com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Scrapping hltv}
|
13
|
+
spec.description = %q{Simple gem to get csgo pro infos}
|
14
|
+
spec.homepage = "https://github.com/RonanLOUARN/hltv-scrapper-light"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
if spec.respond_to?(:metadata)
|
20
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
21
|
+
else
|
22
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
23
|
+
"public gem pushes."
|
24
|
+
end
|
25
|
+
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
27
|
+
f.match(%r{^(test|spec|features)/})
|
28
|
+
end
|
29
|
+
spec.bindir = "exe"
|
30
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
|
+
spec.require_paths = ["lib"]
|
32
|
+
|
33
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
34
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
35
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
36
|
+
end
|
data/lib/.DS_Store
ADDED
Binary file
|
@@ -0,0 +1,269 @@
|
|
1
|
+
require "hltv_scrapping/version"
|
2
|
+
|
3
|
+
module HltvScrapping
|
4
|
+
class Call
|
5
|
+
def self.generate_models
|
6
|
+
puts "Génération des team"
|
7
|
+
# team generator
|
8
|
+
system("rails g model team name maps:integer kd_diff:integer kd:float rating:float link")
|
9
|
+
|
10
|
+
puts "Génération des matchs"
|
11
|
+
# matches generator
|
12
|
+
system("rails g model match event date:datetime map key link")
|
13
|
+
|
14
|
+
puts "Génération des participations aux matchs"
|
15
|
+
# matches generator
|
16
|
+
system("rails g model participation team:references match:references score:integer win:boolean")
|
17
|
+
|
18
|
+
puts "Génération des map stats"
|
19
|
+
# team map stats generator
|
20
|
+
system("rails g model map_stat map team:references win draw losses win_rate:integer total_rounds:integer round_win_after_first_kill_percent:integer round_win_after_first_death_percent:integer")
|
21
|
+
|
22
|
+
puts "Génération de player"
|
23
|
+
# players generator
|
24
|
+
system("rails g model player link team:references maps_played:integer name actif:boolean")
|
25
|
+
|
26
|
+
puts "Génération de player global stat"
|
27
|
+
#player globals stats generator
|
28
|
+
system("rails g model player_global_stat total_kills:integer headshot_percent:float total_deaths:integer kd:float damage_per_round:float grenade_dmg_per_round:float maps_played:integer rounds_played:integer kills_per_round:float assist_per_round:float death_per_round:float saved_by_teammates_per_round:float saved_teammates_per_round:float player:references")
|
29
|
+
|
30
|
+
puts "Génération de participation de joueurs"
|
31
|
+
#Play participation match generator
|
32
|
+
system("rails g model player_participation_match player:references kills:integer head_shots:integer assists:integer deaths:integer kast_percent:float kd_diff:integer average_damage_per_rounds:float fk_diff:integer participation:references")
|
33
|
+
|
34
|
+
#migrate migrations
|
35
|
+
system("rails db:migrate")
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
def self.get_all_teams
|
40
|
+
link = "https://www.hltv.org/stats/teams"
|
41
|
+
html_file = open(link)
|
42
|
+
html_doc = Nokogiri::HTML(html_file)
|
43
|
+
|
44
|
+
html_doc.search('table.stats-table > tbody > tr').each do |element|
|
45
|
+
team_link = element.search('td > a').map { |link| link['href'] }.first
|
46
|
+
match_link = team_link.split("/").insert(3, "matches").join("/")
|
47
|
+
team_name = element.search('td.teamCol > a').map { |link| link.text }.first
|
48
|
+
team_maps_played = element.search('td.statsDetail').map { |link| link.text }.first
|
49
|
+
kd_diff = element.search('td.kdDiffCol').map { |link| link.text }.first.to_i
|
50
|
+
kd = element.search('td.statsDetail').map { |link| link.text }.last.to_f
|
51
|
+
rating = element.search('td.ratingCol').map { |link| link.text }.first.to_f
|
52
|
+
new_team = Team.find_or_create_by(name: team_name, link: match_link, maps: team_maps_played, kd_diff: kd_diff, kd: kd, rating: rating)
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.get_all_matches
|
58
|
+
teams = Team.all
|
59
|
+
teams.each do |team|
|
60
|
+
link = "https://www.hltv.org#{team.link}"
|
61
|
+
html_file = open(link)
|
62
|
+
html_doc = Nokogiri::HTML(html_file)
|
63
|
+
|
64
|
+
html_doc.search('table.stats-table > tbody > tr').each do |element|
|
65
|
+
match_date = Date.parse(element.search('td.time > a').first.text)
|
66
|
+
link = element.search('td.time > a').map { |link| link['href'] }.first
|
67
|
+
event = element.search('td.gtSmartphone-only > a').first.text
|
68
|
+
map = element.search('td.statsMapPlayed > span').first.text.downcase
|
69
|
+
score = element.search('td.gtSmartphone-only > span').last.text
|
70
|
+
|
71
|
+
team2 = Team.find_by_name(element.search('td:nth-child(4) > a').last.text)
|
72
|
+
score1 = score.split("-").first.to_i
|
73
|
+
score2 = score.split("-").last.to_i
|
74
|
+
if team2.present?
|
75
|
+
team_uniq_id = team.id > team2.id ? "#{team2.id}#{team.id}" : "#{team.id}#{team2.id}"
|
76
|
+
uniq_key = "#{team_uniq_id}#{match_date}#{map}"
|
77
|
+
if Match.where(key: uniq_key).empty?
|
78
|
+
#match part
|
79
|
+
team_match = Match.find_or_create_by(event: event, date: match_date, map: map, key: uniq_key, link: link)
|
80
|
+
#participation part
|
81
|
+
p1 = Participation.find_or_create_by(team: team, match: team_match, score: score1, win: score1 > score2 )
|
82
|
+
p2 = Participation.find_or_create_by(team: team2, match: team_match, score: score2, win: score2 > score1 )
|
83
|
+
|
84
|
+
puts "Match saved"
|
85
|
+
else
|
86
|
+
puts "#{element.search('td:nth-child(4) > a').last.text} doesn't exist in db"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def self.update_stats
|
94
|
+
Team.all.each do |team|
|
95
|
+
maps_link = "https://www.hltv.org#{team.link.gsub("matches", "maps")}"
|
96
|
+
html_file = open(maps_link)
|
97
|
+
html_doc = Nokogiri::HTML(html_file)
|
98
|
+
|
99
|
+
html_doc.search('div.two-grid > div.col').each do |element|
|
100
|
+
map_name = element.search('div.map-pool > a > div > div').text
|
101
|
+
wdl = element.search('div.stats-rows > div > span')[1].text unless element.search('div.stats-rows > div > span')[1].nil?
|
102
|
+
win = wdl.split("/").first.to_i unless wdl.nil?
|
103
|
+
draw = wdl.split("/")[1].to_i unless wdl.nil?
|
104
|
+
losses = wdl.split("/").last.to_i unless wdl.nil?
|
105
|
+
|
106
|
+
win_rate = element.search('div.stats-rows > div > span')[3].text.to_i unless element.search('div.stats-rows > div > span')[3].nil?
|
107
|
+
total_rounds = element.search('div.stats-rows > div > span')[5].text.to_i unless element.search('div.stats-rows > div > span')[5].nil?
|
108
|
+
round_win_after_first_kill = element.search('div.stats-rows > div > span')[7].text.to_i unless element.search('div.stats-rows > div > span')[7].nil?
|
109
|
+
round_win_after_first_death = element.search('div.stats-rows > div > span')[9].text.to_i unless element.search('div.stats-rows > div > span')[9].nil?
|
110
|
+
existant_stats = MapStat.where(map: map_name, team: team)
|
111
|
+
|
112
|
+
if existant_stats.any?
|
113
|
+
existant_stats.first.update(win: win, draw: draw, losses: losses, win_rate: win_rate, total_rounds: total_rounds, round_win_after_first_kill_percent: round_win_after_first_kill, round_win_after_first_death_percent: round_win_after_first_death)
|
114
|
+
else
|
115
|
+
MapStat.find_or_create_by(map: map_name, team: team ,win: win, draw: draw, losses: losses, win_rate: win_rate, total_rounds: total_rounds, round_win_after_first_kill_percent: round_win_after_first_kill, round_win_after_first_death_percent: round_win_after_first_death)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.get_players
|
122
|
+
Team.all.each do |team|
|
123
|
+
new_link = team.link.split("/")
|
124
|
+
new_link.reject!{ |el| el == "matches"}
|
125
|
+
players_link = "https://www.hltv.org#{new_link.join("/")}"
|
126
|
+
puts players_link
|
127
|
+
html_file = open(players_link)
|
128
|
+
html_doc = Nokogiri::HTML(html_file)
|
129
|
+
|
130
|
+
html_doc.search('div:nth-child(9) > div > .teammate-info').each do |element|
|
131
|
+
player_name = element.search("a > div.text-ellipsis").text
|
132
|
+
player_maps = element.search("span").text.split(" ").first.to_i
|
133
|
+
player_link = element.search("a").map { |link| link['href'] }.first
|
134
|
+
Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: true)
|
135
|
+
end
|
136
|
+
|
137
|
+
html_doc.search('div:nth-child(12) > div > .teammate-info').each do |element|
|
138
|
+
player_name = element.search("a > div.text-ellipsis").text
|
139
|
+
player_maps = element.search("span").text.split(" ").first.to_i
|
140
|
+
player_link = element.search("a").map { |link| link['href'] }.first
|
141
|
+
Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: false)
|
142
|
+
end
|
143
|
+
|
144
|
+
html_doc.search('div:nth-child(15) > div > .teammate-info').each do |element|
|
145
|
+
player_name = element.search("a > div.text-ellipsis").text
|
146
|
+
player_maps = element.search("span").text.split(" ").first.to_i
|
147
|
+
player_link = element.search("a").map { |link| link['href'] }.first
|
148
|
+
Player.find_or_create_by(name: player_name, maps_played: player_maps.to_i, link: player_link, team: team, actif: false)
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def self.get_players_global_stats
|
155
|
+
Player.all.each do |player|
|
156
|
+
player_link = "https://www.hltv.org#{player.link}"
|
157
|
+
html_file = open(player_link)
|
158
|
+
html_doc = Nokogiri::HTML(html_file)
|
159
|
+
|
160
|
+
html_doc.search('div.contentCol > div > div.statistics > div > div:nth-child(1)').each do |element|
|
161
|
+
@total_kills = element.search('div:nth-child(1) > span:nth-child(2)').text.to_i
|
162
|
+
@headshot_percent = element.search('div:nth-child(2) > span:nth-child(2)').text.to_f
|
163
|
+
@total_deaths = element.search('div:nth-child(3) > span:nth-child(2)').text.to_i
|
164
|
+
@kd = element.search('div:nth-child(4) > span:nth-child(2)').text.to_f
|
165
|
+
@damage_per_round = element.search('div:nth-child(5) > span:nth-child(2)').text.to_f
|
166
|
+
@grenade_dmg_per_round = element.search('div:nth-child(6) > span:nth-child(2)').text.to_f
|
167
|
+
@maps_played = element.search('div:nth-child(7) > span:nth-child(2)').text.to_i
|
168
|
+
end
|
169
|
+
|
170
|
+
html_doc.search('div.contentCol > div > div.statistics > div > div:nth-child(2)').each do |element|
|
171
|
+
@rounds_played = element.search('div:nth-child(1) > span:nth-child(2)').text.to_i
|
172
|
+
@kills_per_round = element.search('div:nth-child(2) > span:nth-child(2)').text.to_f
|
173
|
+
@assist_per_round = element.search('div:nth-child(3) > span:nth-child(2)').text.to_f
|
174
|
+
@death_per_round = element.search('div:nth-child(4) > span:nth-child(2)').text.to_f
|
175
|
+
@saved_by_teammates_per_round = element.search('div:nth-child(5) > span:nth-child(2)').text.to_f
|
176
|
+
@saved_teammates_per_round = element.search('div:nth-child(6) > span:nth-child(2)').text.to_f
|
177
|
+
end
|
178
|
+
|
179
|
+
hsh_player_stats = {
|
180
|
+
total_kills: @total_kills,
|
181
|
+
headshot_percent: @headshot_percent,
|
182
|
+
total_deaths: @total_deaths,
|
183
|
+
kd: @kd,
|
184
|
+
damage_per_round: @damage_per_round,
|
185
|
+
grenade_dmg_per_round: @grenade_dmg_per_round,
|
186
|
+
maps_played: @maps_played,
|
187
|
+
rounds_played: @rounds_played,
|
188
|
+
kills_per_round: @kills_per_round,
|
189
|
+
assist_per_round: @assist_per_round,
|
190
|
+
death_per_round: @death_per_round,
|
191
|
+
saved_by_teammates_per_round: @saved_by_teammates_per_round,
|
192
|
+
saved_teammates_per_round: @saved_teammates_per_round,
|
193
|
+
player: player
|
194
|
+
}
|
195
|
+
PlayerGlobalStat.find_or_create_by(hsh_player_stats)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def self.get_player_participation
|
200
|
+
Match.all.each do |match|
|
201
|
+
match_link = "https://www.hltv.org#{match.link}"
|
202
|
+
puts match_link
|
203
|
+
html_file = open(match_link)
|
204
|
+
# html_file = open(match_link)
|
205
|
+
html_doc = Nokogiri::HTML(html_file)
|
206
|
+
|
207
|
+
html_doc.search('div.colCon > div.contentCol > div > table:nth-of-type(1) > tbody > tr').each do |element|
|
208
|
+
player1 = Player.select{|player| player.link == element.search('td:nth-child(1) > a').map { |link| link['href'] }.first && (match.participations.map(&:team_id).include? player.team_id) }.first
|
209
|
+
kills1 = element.search('td:nth-child(2)').text.split("(").first.to_i
|
210
|
+
head_shots1 = element.search('td:nth-child(2)').text.split("(").last.gsub(")","").to_i
|
211
|
+
assists1 = element.search('td:nth-child(3)').text.to_i
|
212
|
+
deaths1 = element.search('td:nth-child(4)').text.to_i
|
213
|
+
kast1 = element.search('td:nth-child(5)').text.to_f
|
214
|
+
kd_diff1 = element.search('td:nth-child(6)').text.to_f
|
215
|
+
average_damage_per_round1 = element.search('td:nth-child(7)').text.to_f
|
216
|
+
fk_diff1 = element.search('td:nth-child(8)').text.to_f
|
217
|
+
if player1
|
218
|
+
participation1 = match.participations.where(team_id: player1.team.id).first
|
219
|
+
# byebug
|
220
|
+
pp1 = PlayerParticipationMatch.find_or_create_by(
|
221
|
+
player: player1,
|
222
|
+
kills: kills1,
|
223
|
+
head_shots: head_shots1,
|
224
|
+
assists: assists1,
|
225
|
+
deaths: deaths1,
|
226
|
+
kast_percent: kast1,
|
227
|
+
kd_diff: kd_diff1,
|
228
|
+
average_damage_per_rounds: average_damage_per_round1,
|
229
|
+
fk_diff: fk_diff1,
|
230
|
+
participation: participation1
|
231
|
+
)
|
232
|
+
puts "--------------"
|
233
|
+
puts match.id
|
234
|
+
puts pp1.errors.inspect
|
235
|
+
puts "--------------"
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
html_doc.search('div.colCon > div.contentCol > div > table:nth-of-type(2) > tbody > tr').each do |element|
|
240
|
+
player = Player.all.select{|player| player.link == element.search('td:nth-child(1) > a').map { |link| link['href'] }.first && (match.participations.map(&:team_id).include? player.team_id) }.first
|
241
|
+
kills = element.search('td:nth-child(2)').text.split("(").first.to_i
|
242
|
+
head_shots = element.search('td:nth-child(2)').text.split("(").last.gsub(")","").to_i
|
243
|
+
assists = element.search('td:nth-child(3)').text.to_i
|
244
|
+
deaths = element.search('td:nth-child(4)').text.to_i
|
245
|
+
kast = element.search('td:nth-child(5)').text.to_f
|
246
|
+
kd_diff = element.search('td:nth-child(6)').text.to_f
|
247
|
+
average_damage_per_round = element.search('td:nth-child(7)').text.to_f
|
248
|
+
fk_diff = element.search('td:nth-child(8)').text.to_f
|
249
|
+
if player
|
250
|
+
participation = match.participations.where(team_id: player.team.id).first
|
251
|
+
pp2 = PlayerParticipationMatch.find_or_create_by(
|
252
|
+
player: player,
|
253
|
+
kills: kills,
|
254
|
+
head_shots: head_shots,
|
255
|
+
assists: assists,
|
256
|
+
deaths: deaths,
|
257
|
+
kast_percent: kast,
|
258
|
+
kd_diff: kd_diff,
|
259
|
+
average_damage_per_rounds: average_damage_per_round,
|
260
|
+
fk_diff: fk_diff,
|
261
|
+
participation: participation
|
262
|
+
)
|
263
|
+
puts pp2.errors.inspect
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
task :generate_models => :environment do
|
2
|
+
puts "Génération des team"
|
3
|
+
# team generator
|
4
|
+
system("rails g model team name maps:integer kd_diff:integer kd:float rating:float link")
|
5
|
+
|
6
|
+
puts "Généation des matchs"
|
7
|
+
# matches generator
|
8
|
+
system("rails g model match event date:datetime map key link")
|
9
|
+
|
10
|
+
puts "Génération des map stats"
|
11
|
+
# team map stats generator
|
12
|
+
system("rails g model map_stat map team:references win draw losses win_rate:integer total_rounds:integer round_win_after_first_kill_percent:integer round_win_after_first_death_percent:integer")
|
13
|
+
|
14
|
+
puts "Génération de player"
|
15
|
+
# players generator
|
16
|
+
system("rails g model player link team:references maps_played:integer name actif:boolean")
|
17
|
+
|
18
|
+
puts "Génération de player global stat"
|
19
|
+
#player globals stats generator
|
20
|
+
system("rails g model player_global_stat total_kills:integer headshot_percent:float total_deaths:integer kd:float damage_per_round:float grenade_dmg_per_round:float maps_played:integer rounds_played:integer kills_per_round:float assist_per_round:float death_per_round:float saved_by_teammates_per_round:float saved_teammates_per_round:float player:references")
|
21
|
+
|
22
|
+
puts "Génération de participation de joueurs"
|
23
|
+
#Play participation match generator
|
24
|
+
system("rails g model player_participation_match player:references kills:integer head_shots:integer assists:integer deaths:integer kast_percent:float kd_diff:integer average_damage_per_rounds:float fk_diff:integer participation:references")
|
25
|
+
|
26
|
+
#migrate migrations
|
27
|
+
system("rails db:migrate")
|
28
|
+
end
|
29
|
+
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hltv_scrapping
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.8
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- ronan louarn
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.14'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
description: Simple gem to get csgo pro infos
|
56
|
+
email:
|
57
|
+
- ronan33720@hotmail.Com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".DS_Store"
|
63
|
+
- Gemfile
|
64
|
+
- Gemfile.lock
|
65
|
+
- LICENSE.txt
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- bin/console
|
69
|
+
- bin/setup
|
70
|
+
- hltv_scrapping.gemspec
|
71
|
+
- lib/.DS_Store
|
72
|
+
- lib/hltv_scrapping.rb
|
73
|
+
- lib/hltv_scrapping/version.rb
|
74
|
+
- lib/tasks/generate_models.rake
|
75
|
+
homepage: https://github.com/RonanLOUARN/hltv-scrapper-light
|
76
|
+
licenses:
|
77
|
+
- MIT
|
78
|
+
metadata:
|
79
|
+
allowed_push_host: https://rubygems.org
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options: []
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - ">="
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
requirements: []
|
95
|
+
rubyforge_project:
|
96
|
+
rubygems_version: 2.5.2
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: Scrapping hltv
|
100
|
+
test_files: []
|