adops_report_scrapper 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +3 -0
  5. data/CODE_OF_CONDUCT.md +13 -0
  6. data/Gemfile +18 -0
  7. data/LICENSE.txt +21 -0
  8. data/README.md +51 -0
  9. data/Rakefile +152 -0
  10. data/adops_report_scrapper.gemspec +24 -0
  11. data/bin/console +14 -0
  12. data/bin/setup +7 -0
  13. data/lib/adops_report_scrapper.rb +38 -0
  14. data/lib/adops_report_scrapper/adaptv_client.rb +80 -0
  15. data/lib/adops_report_scrapper/adforge_client.rb +35 -0
  16. data/lib/adops_report_scrapper/adiply_client.rb +49 -0
  17. data/lib/adops_report_scrapper/adsense_client.rb +55 -0
  18. data/lib/adops_report_scrapper/adsupply_client.rb +27 -0
  19. data/lib/adops_report_scrapper/adx_client.rb +55 -0
  20. data/lib/adops_report_scrapper/base_client.rb +74 -0
  21. data/lib/adops_report_scrapper/brightroll_client.rb +47 -0
  22. data/lib/adops_report_scrapper/browsi_client.rb +44 -0
  23. data/lib/adops_report_scrapper/contentad_client.rb +91 -0
  24. data/lib/adops_report_scrapper/conversant_client.rb +66 -0
  25. data/lib/adops_report_scrapper/criteo_client.rb +25 -0
  26. data/lib/adops_report_scrapper/facebookaudience_client.rb +56 -0
  27. data/lib/adops_report_scrapper/gcs_client.rb +56 -0
  28. data/lib/adops_report_scrapper/liveintent_client.rb +90 -0
  29. data/lib/adops_report_scrapper/marfeel_client.rb +59 -0
  30. data/lib/adops_report_scrapper/nativo_client.rb +60 -0
  31. data/lib/adops_report_scrapper/netseer_client.rb +73 -0
  32. data/lib/adops_report_scrapper/openx_client.rb +161 -0
  33. data/lib/adops_report_scrapper/revcontent_client.rb +51 -0
  34. data/lib/adops_report_scrapper/sonobi_client.rb +91 -0
  35. data/lib/adops_report_scrapper/springserve_client.rb +52 -0
  36. data/lib/adops_report_scrapper/tremor_client.rb +52 -0
  37. data/lib/adops_report_scrapper/triplelift_client.rb +72 -0
  38. data/lib/adops_report_scrapper/version.rb +3 -0
  39. data/secret.sample.yml +77 -0
  40. data/tmp/.keep +0 -0
  41. metadata +117 -0
@@ -0,0 +1,161 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::OpenxClient < AdopsReportScrapper::BaseClient
5
+ REPORT_NAME = 'Ad Server Report for adops_report_scrapper'
6
+
7
+ private
8
+
9
+ def login
10
+ @client.driver.resize 1920, 700
11
+ fail 'please specify openx account prefix' unless @options['account_prefix']
12
+ @account_prefix = @options['account_prefix']
13
+ @client.visit "http://#{@account_prefix}.openx.net/"
14
+ @client.fill_in 'Email', :with => @login
15
+ @client.fill_in 'Password', :with => @secret
16
+ @client.click_button 'Submit'
17
+ begin
18
+ @client.find :xpath, '//*[text()="Reports"]'
19
+ rescue Exception => e
20
+ raise e, 'Openx login error'
21
+ end
22
+ end
23
+
24
+ def scrap
25
+ request_report
26
+ end
27
+
28
+ def request_report
29
+ @client.visit 'http://cmci-ui.openx.net/#/reports?tab=my_reports'
30
+ sleep 5
31
+
32
+ begin
33
+ tries ||= 6
34
+ @client.find(:css, '#report_frame')
35
+ rescue Exception => e
36
+ retry unless (tries -= 1).zero?
37
+ end
38
+
39
+ create_report_if_not_exist
40
+
41
+ @client.within_frame @client.find(:css, '#report_frame') do
42
+ @client.find(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").click
43
+
44
+ begin
45
+ tries ||= 6
46
+ @client.find(:css, '.myFrame')
47
+ rescue Exception => e
48
+ retry unless (tries -= 1).zero?
49
+ end
50
+
51
+ @client.within_frame @client.find(:css, '.myFrame') do
52
+ begin
53
+ tries ||= 18
54
+ @client.find(:xpath, '//option[text()="500"]')
55
+ rescue Exception => e
56
+ retry unless (tries -= 1).zero?
57
+ end
58
+
59
+ @client.find(:xpath, '//option[text()="500"]').select_option
60
+ extract_data_from_report
61
+ end
62
+ end
63
+ sleep 5
64
+ end
65
+
66
+ def create_report_if_not_exist
67
+ @client.within_frame @client.find(:css, '#report_frame') do
68
+ ready_elem = nil
69
+ begin
70
+ tries ||= 6
71
+ ready_elem = @client.find(:xpath, '//*[text()="Preconfigured Reports"]')
72
+ rescue Exception => e
73
+ retry unless (tries -= 1).zero?
74
+ end
75
+
76
+ fail 'openx report page not ready' unless ready_elem
77
+
78
+ if @client.find_all(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").count == 0
79
+
80
+ # create report if not exist
81
+ @client.find(:xpath, '//*[contains(text(),"Create Report")]').click
82
+ @client.find(:xpath, '//li/*[text()="Ad Server Report"]').click
83
+
84
+ begin
85
+ tries ||= 6
86
+ @client.find(:css, '.myFrame')
87
+ rescue Exception => e
88
+ retry unless (tries -= 1).zero?
89
+ end
90
+
91
+ @client.within_frame @client.find(:css, '.myFrame') do
92
+
93
+ @client.find(:xpath, '//*[text()="Last Seven Days"]').click
94
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
95
+ @client.find(:xpath, '//*[@value="Next"]').trigger('click')
96
+ sleep 2
97
+
98
+ @client.find(:xpath, '//*[text()="Ad Unit"]').click
99
+ @client.find(:xpath, '//img[../../td//*[text()="User"]]').click
100
+ @client.find(:xpath, '//*[text()="Country"]').click
101
+ @client.find(:xpath, '//*[text()="Device Category"]').click
102
+ @client.find(:xpath, '//*[@value="Next"]').trigger('click')
103
+ sleep 2
104
+
105
+ @client.find(:xpath, '//*[text()="Paid Impressions"]').click
106
+ @client.find(:xpath, '//*[text()="Impressions Delivered"]').click
107
+ @client.find(:xpath, '//*[text()="Ad Requests"]').click
108
+ @client.find(:xpath, '//img[../../td//*[text()="Revenue"]]').click
109
+ @client.find(:xpath, '//*[text()="Publisher Revenue"]').click
110
+ @client.find(:xpath, '//img[../../td//*[text()="Clicks"]]').click
111
+ @client.find(:xpath, '//div[text()="Clicks"]').click
112
+ @client.find(:xpath, '//*[@value="Next"]').trigger('click')
113
+ sleep 2
114
+
115
+ @client.find(:xpath, '//*[@value="Save"]').trigger('click')
116
+
117
+ begin
118
+ tries ||= 6
119
+ @client.find(:xpath, '//*[text()="Save Report"]')
120
+ rescue Exception => e
121
+ retry unless (tries -= 1).zero?
122
+ end
123
+
124
+ @client.fill_in 'saveAsReportName', :with => REPORT_NAME
125
+ @client.find(:xpath, '//*[@value="Save Report"]').trigger('click')
126
+
127
+ begin
128
+ tries ||= 6
129
+ @client.find(:xpath, '//*[text()="Report Saved"]')
130
+ rescue Exception => e
131
+ retry unless (tries -= 1).zero?
132
+ end
133
+
134
+ @client.find(:xpath, '//*[text()="Back to My Reports"]').trigger('click')
135
+ end
136
+ sleep 2
137
+ end
138
+ end
139
+ end
140
+
141
+ def extract_data_from_report
142
+ @data = []
143
+ loop do
144
+ 18.times do
145
+ sleep 10
146
+ break if @client.find_all(:xpath, '//table[@id="table_UniqueReportID"]/*/tr').count > 0
147
+ end
148
+ rows = @client.find_all :xpath, '//table[@id="table_UniqueReportID"]/*/tr'
149
+ rows = rows.to_a
150
+ header = rows.shift
151
+ if @data.count == 0
152
+ n_header = header.find_css('td,th').map { |td| td.visible_text }
153
+ @data << n_header
154
+ end
155
+ @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }.reject { |row| row[0] == 'Total' }
156
+ pagee = @client.find(:xpath, '//*[contains(text(),"Showing rows")]').text.match(/-(\d+) of (\d+)\./).captures
157
+ break if pagee[0] == pagee[1]
158
+ @client.find_all(:css, '#paginationNext').first.trigger('click')
159
+ end
160
+ end
161
+ end
@@ -0,0 +1,51 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::RevcontentClient < AdopsReportScrapper::BaseClient
5
+ private
6
+
7
+ def login
8
+ @client.visit 'https://www.revcontent.com/login'
9
+ @client.fill_in 'name', :with => @login
10
+ @client.fill_in 'password', :with => @secret
11
+ @client.click_button 'Sign In'
12
+ begin
13
+ @client.find :xpath, '//*[contains(text(),"Widgets")]'
14
+ rescue Exception => e
15
+ raise e, 'Revcontent login error'
16
+ end
17
+ end
18
+
19
+ def scrap
20
+ @client.find(:xpath, '//*[contains(text(),"Widgets")]').click
21
+ @client.find(:css, '.fa.fa-calendar').click
22
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
23
+ @data = []
24
+ %w(desktoplg desktop tablet phone unknown).each do |device|
25
+ request_report(device)
26
+ extract_data_from_report(device)
27
+ end
28
+ end
29
+
30
+ def request_report(device)
31
+ @client.find(:css, '.fa.fa-desktop').click
32
+ @client.find(:xpath, '//*[text()="All Devices"]').click
33
+ @client.find(:css, '.fa.fa-desktop').click
34
+ @client.check device
35
+ end
36
+
37
+ def extract_data_from_report(device)
38
+ rows = @client.find_all :xpath, '//table/*/tr'
39
+ rows = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
40
+ header = rows.shift
41
+ rows.shift
42
+ if @data.count == 0
43
+ header.unshift 'Device'
44
+ @data << header
45
+ end
46
+ rows.each do |row|
47
+ row.unshift device
48
+ end
49
+ @data += rows
50
+ end
51
+ end
@@ -0,0 +1,91 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::SonobiClient < AdopsReportScrapper::BaseClient
5
+ private
6
+
7
+ def login
8
+ @client.visit 'https://jetstream.sonobi.com/welcome/login.php'
9
+ @client.fill_in 'user name', :with => @login
10
+ @client.fill_in 'password', :with => @secret
11
+ @client.click_button 'Submit'
12
+ begin
13
+ @client.find :xpath, '//*[text()="dashboard"]'
14
+ rescue Exception => e
15
+ raise e, 'Sonobi login error'
16
+ end
17
+ end
18
+
19
+ def scrap
20
+ @data = []
21
+ request_report(:us)
22
+ extract_data_from_report(:us)
23
+ request_report(:intl)
24
+ extract_data_from_report(:intl)
25
+ end
26
+
27
+ def request_report(country)
28
+ is_us = country == :us
29
+ # all sites
30
+ @client.find(:xpath, '//*[text()="Reports"]').click
31
+ sleep 2
32
+ @client.find(:xpath, '//div[@name="_siteid"]').click
33
+ sleep 2
34
+ @client.find(:xpath, '//*[text()="Select All"]').click
35
+
36
+ # select country
37
+ @client.find(:xpath, '//*[text()="Add New Filter"]').click
38
+ @client.find_all(:xpath, '//option[text()="Country"]').last.select_option
39
+ sleep 2
40
+ @client.find(:xpath, '//div[@name="_country"]').click
41
+ sleep 2
42
+ if is_us
43
+ @client.find(:xpath, '//*[text()="United States"]').click
44
+ else
45
+ @client.find(:xpath, '//*[text()="Select All"]').click
46
+ @client.find(:xpath, '//*[@class="remove_icon"][../*[text()="United States"]]').click
47
+ end
48
+
49
+ # check group by
50
+ @client.check 'Date'
51
+ @client.check 'Placement'
52
+ @client.check 'Site'
53
+ @client.check 'Clicks'
54
+ @client.check 'Views'
55
+ @client.check 'Unfilled Impressions'
56
+ @client.check 'Device Type'
57
+
58
+ @client.click_button 'Run Report'
59
+ sleep 1
60
+ wait_for_spin
61
+ end
62
+
63
+ def extract_data_from_report(country)
64
+ rows = @client.find_all :xpath, '//*[@class="reports_tab_item_body"]//table/*/tr'
65
+ rows = rows.to_a
66
+ header = rows.shift
67
+ if @data.count == 0
68
+ n_header = header.find_css('td,th').map { |td| td.visible_text }
69
+ n_header << 'Country'
70
+ @data << n_header
71
+ end
72
+ rows.shift
73
+ @data += rows.map do |row|
74
+ n_row = row.find_css('td,th').map { |td| td.visible_text }
75
+ n_row << country.to_s.upcase
76
+ n_row
77
+ end
78
+ end
79
+
80
+ def wait_for_spin
81
+ 30.times do |_i| # wait 5 min
82
+ begin
83
+ @client.find(:css, '.circle xlarge')
84
+ rescue Exception => e
85
+ break
86
+ end
87
+ sleep 10
88
+ end
89
+ sleep 5
90
+ end
91
+ end
@@ -0,0 +1,52 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::SpringserveClient < AdopsReportScrapper::BaseClient
5
+ private
6
+
7
+ def login
8
+ @client.visit 'http://video.springserve.com/'
9
+ @client.fill_in 'Email', :with => @login
10
+ @client.fill_in 'Password', :with => @secret
11
+ @client.click_button 'Log in'
12
+ begin
13
+ @client.find :xpath, '//*[contains(text(),"Reporting")]'
14
+ rescue Exception => e
15
+ raise e, 'Springserve login error'
16
+ end
17
+ end
18
+
19
+ def scrap
20
+ request_report
21
+ extract_data_from_report
22
+ end
23
+
24
+ def request_report
25
+ @client.find(:xpath, '//*[contains(text(),"Reporting")]').click
26
+ @client.find(:xpath, '//*[contains(text(),"Create Reports")]').click
27
+
28
+ # select date
29
+ @client.find(:css, '#date_range_chosen').click
30
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
31
+
32
+ @client.find(:css, '#dimensions_chosen').click
33
+ @client.find(:xpath, '//*[text()="Country"]').click
34
+
35
+ @client.find(:xpath, '//*[@value="Run Report"]').click
36
+
37
+ 30.times do |_i| # wait 5 min
38
+ begin
39
+ @client.find(:css, '#spinner_image')
40
+ rescue Exception => e
41
+ break
42
+ end
43
+ sleep 10
44
+ end
45
+ sleep 10
46
+ end
47
+
48
+ def extract_data_from_report
49
+ rows = @client.find_all :xpath, '//table[1]/*/tr'
50
+ @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }.reject { |row| row[0] == 'Total' }
51
+ end
52
+ end
@@ -0,0 +1,52 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
5
+ private
6
+
7
+ def login
8
+ @client.visit 'https://console.tremorhub.com/ssp'
9
+ @client.fill_in 'username', :with => @login
10
+ @client.fill_in 'password', :with => @secret
11
+ @client.click_button 'Sign In'
12
+ begin
13
+ @client.find :xpath, '//*[text()="REPORTS"]'
14
+ rescue Exception => e
15
+ raise e, 'Tremor login error'
16
+ end
17
+ end
18
+
19
+ def scrap
20
+ request_report
21
+ extract_data_from_report
22
+ end
23
+
24
+ def request_report
25
+ @client.find(:xpath, '//*[text()="REPORTS"]').click
26
+ @client.find(:xpath, '//*[text()="Hierarchical"]').click
27
+ sleep 1
28
+ # select group by
29
+ @client.find(:css, '#hierarchicalReportsGroupBys').click
30
+ @client.find(:xpath, '//*[text()="AdUnit"]').click
31
+ @client.find(:css, '#hierarchicalReportsGroupBys').click
32
+ @client.find(:xpath, '//*[text()="Country"]').click
33
+ # select date
34
+ @client.find(:css, '#hierarchicalReportsDateRange').click
35
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
36
+ @client.click_button 'Run'
37
+ sleep 10
38
+ 30.times do |_i| # wait 5 min
39
+ begin
40
+ @client.find(:xpath, '//*[text()="Please Hold"]')
41
+ rescue Exception => e
42
+ break
43
+ end
44
+ sleep 10
45
+ end
46
+ end
47
+
48
+ def extract_data_from_report
49
+ rows = @client.find_all :xpath, '//table/*/tr'
50
+ @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
51
+ end
52
+ end
@@ -0,0 +1,72 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+
4
+ class AdopsReportScrapper::TripleliftClient < AdopsReportScrapper::BaseClient
5
+ private
6
+
7
+ def login
8
+ @client.visit 'https://console.triplelift.com/login'
9
+ @client.fill_in 'Email', :with => @login
10
+ @client.fill_in 'Password', :with => @secret
11
+ @client.click_button 'Sign in'
12
+ begin
13
+ @client.find :xpath, '//*[text()="Reporting"]'
14
+ rescue Exception => e
15
+ raise e, 'Triplelift login error'
16
+ end
17
+ end
18
+
19
+ def scrap
20
+ @date_str = @date.strftime('%B %d, %Y')
21
+ @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
22
+ @publishers = @client.find_all(:xpath, '//*[@ng-click="selectPub(pub)"]').to_a.map { |pub_elem| pub_elem.text(:all) }
23
+ @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
24
+ @data = []
25
+ while @publishers.count > 0
26
+ extract_data(@publishers.shift)
27
+ end
28
+ end
29
+
30
+ def extract_data(publisher)
31
+ @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
32
+ index = -1 - @publishers.count(publisher)
33
+ sleep 1
34
+ @client.find_all(:xpath, "//*[text()=\"#{publisher}\"]")[index].click
35
+ @client.find(:xpath, '//*[text()="Reporting"]').click
36
+ sleep 2
37
+
38
+ return if @client.find_all(:xpath, '//*[text()="No data available for selected date range"]').count > 0
39
+
40
+ @client.find(:xpath, '//*[@model="startDate"]//input').set @date_str
41
+ sleep 1
42
+ @client.find(:xpath, '//*[@model="endDate"]//input').set @date_str
43
+ sleep 1
44
+ @client.find(:xpath, '//button[../../div[contains(text(),"Group by")]]').click
45
+ @client.find(:xpath, '//*[text()="Date and Placement"]').click
46
+ wait_for_spin
47
+
48
+ return if @client.find_all(:xpath, '//*[text()="No data available for selected date range"]').count > 0
49
+
50
+ rows = @client.find_all :xpath, '//table/*/tr'
51
+ rows = rows.to_a
52
+ rows.shift
53
+ header = rows.shift
54
+ if @data.count == 0
55
+ n_header = header.find_css('td,th').map { |td| td.visible_text }
56
+ @data << n_header
57
+ end
58
+ @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
59
+ end
60
+
61
+ def wait_for_spin
62
+ 30.times do |_i| # wait 5 min
63
+ begin
64
+ @client.find(:css, '.spinner')
65
+ rescue Exception => e
66
+ break
67
+ end
68
+ sleep 5
69
+ end
70
+ sleep 2
71
+ end
72
+ end