adops_report_scrapper 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +6 -1
- data/lib/adops_report_scrapper/adtechus_client.rb +54 -0
- data/lib/adops_report_scrapper/gcs_client.rb +5 -0
- data/lib/adops_report_scrapper/version.rb +1 -1
- data/lib/adops_report_scrapper.rb +1 -0
- data/secret.sample.yml +4 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: edbe42b9645b997220cd4ac89c2d0473445eb11f
|
4
|
+
data.tar.gz: d3c6d4a8fd8668879b64e6bd0a46e2cc60dd077f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ca089b75280f02a1d9af9a50d7d53b3975e1982541220842acf8955eda005ee13eba45956fd6b11bdcb65342850c3675383d2a141a6c565207818c3608419a70
|
7
|
+
data.tar.gz: e869dfcac65dbcfca94d924105ab4f26364575845a4ad166fb660bf4c73ff0dabeb3510292440e8f8455bc7f04d1ca38644e32648bf9c047e056676e6d57f6fc
|
data/Rakefile
CHANGED
@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
|
|
7
7
|
require 'byebug'
|
8
8
|
|
9
9
|
desc 'Collect all data'
|
10
|
-
task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience] do # openx is the most unstable one, run it first
|
10
|
+
task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus] do # openx is the most unstable one, run it first
|
11
11
|
puts '========== You are all set'
|
12
12
|
end
|
13
13
|
|
@@ -126,6 +126,11 @@ task :facebookaudience do
|
|
126
126
|
save_as_csv :facebookaudience, :facebookaudience
|
127
127
|
end
|
128
128
|
|
129
|
+
desc 'Collect adtechus data'
|
130
|
+
task :adtechus do
|
131
|
+
save_as_csv :adtechus, :adtechus
|
132
|
+
end
|
133
|
+
|
129
134
|
def date
|
130
135
|
@date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
|
131
136
|
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'date'
|
2
|
+
require_relative 'base_client'
|
3
|
+
|
4
|
+
class AdopsReportScrapper::AdtechusClient < AdopsReportScrapper::BaseClient
|
5
|
+
private
|
6
|
+
|
7
|
+
def login
|
8
|
+
@client.visit 'http://marketplace.adtechus.com'
|
9
|
+
@client.fill_in 'Username', :with => @login
|
10
|
+
@client.fill_in 'Password', :with => @secret
|
11
|
+
@client.click_button 'Sign in'
|
12
|
+
begin
|
13
|
+
@client.find :css, '#mainwindow'
|
14
|
+
rescue Exception => e
|
15
|
+
raise e, 'Adtechus login error'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def scrap
|
20
|
+
request_report
|
21
|
+
end
|
22
|
+
|
23
|
+
def request_report
|
24
|
+
@client.visit(@client.find(:css, '#mainwindow')[:src])
|
25
|
+
wait_for_loading
|
26
|
+
report_id = @client.find(:xpath, '//tr[./td/div/span[text()="Placement fill rate report"]]')[:id]
|
27
|
+
report_id = report_id.tr 'row_', ''
|
28
|
+
@client.visit "https://marketplace.adtechus.com/h2/reporting/showReport.do?action=showreportpage._._.#{report_id}"
|
29
|
+
@client.within_frame @client.find(:css, '#reportwindow') do
|
30
|
+
@client.within_frame @client.find(:xpath, '//iframe[@name="uid_2"]') do
|
31
|
+
extract_data_from_report
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def extract_data_from_report
|
37
|
+
rows = @client.find_all :xpath, '//table[@class="table"]/tbody/tr'
|
38
|
+
rows = rows.to_a
|
39
|
+
rows.pop
|
40
|
+
@data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
|
41
|
+
end
|
42
|
+
|
43
|
+
def wait_for_loading
|
44
|
+
18.times do |_i| # wait 3 min
|
45
|
+
begin
|
46
|
+
@client.find(:xpath, '//*[contains(text(),"loading")]')
|
47
|
+
rescue Exception => e
|
48
|
+
break
|
49
|
+
end
|
50
|
+
sleep 3
|
51
|
+
end
|
52
|
+
sleep 1
|
53
|
+
end
|
54
|
+
end
|
@@ -18,6 +18,11 @@ class AdopsReportScrapper::GcsClient < AdopsReportScrapper::BaseClient
|
|
18
18
|
@client.click_button 'Next'
|
19
19
|
@client.fill_in 'Passwd', :with => @secret
|
20
20
|
@client.click_button 'Sign in'
|
21
|
+
# for veirfication
|
22
|
+
# cc = @client.find :xpath, '//*[contains(text(),"the recovery phone")]'
|
23
|
+
# cc.click
|
24
|
+
# @client.fill_in 'Enter phone number', :with => @options[:recovery_phone]
|
25
|
+
# @client.click_button 'Done'
|
21
26
|
begin
|
22
27
|
@client.find :xpath, '//*[text()="Sites"]'
|
23
28
|
rescue Exception => e
|
data/secret.sample.yml
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: adops_report_scrapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stayman Hou
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-08-
|
11
|
+
date: 2016-08-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: httpclient
|
@@ -221,6 +221,7 @@ files:
|
|
221
221
|
- lib/adops_report_scrapper/adiply_client.rb
|
222
222
|
- lib/adops_report_scrapper/adsense_client.rb
|
223
223
|
- lib/adops_report_scrapper/adsupply_client.rb
|
224
|
+
- lib/adops_report_scrapper/adtechus_client.rb
|
224
225
|
- lib/adops_report_scrapper/adx_client.rb
|
225
226
|
- lib/adops_report_scrapper/base_client.rb
|
226
227
|
- lib/adops_report_scrapper/brightroll_client.rb
|