adops_report_scrapper 0.1.69 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b57c737ccfc9f5d880ce391d9f355dedaf8bbc70
4
- data.tar.gz: a5f2495fa1de0149097f909691a285aa14720bdc
3
+ metadata.gz: ddb35e06ab9f273a789732d953f60131dc470ad9
4
+ data.tar.gz: 3f986c6d5bf899335bcfa705ec056786b8a5859e
5
5
  SHA512:
6
- metadata.gz: cd3b2a29fb54892ee2dfbec9731ee2c5cc72b9eeb4c54d82d15b2972427761e0504ba4be9e1cf49a2c186bb086b1ffad3b4102da97a42bce01c54a638f75fadf
7
- data.tar.gz: 822e3682bafe46d6b90b2b9a5f73094245d5f1c877890134a7d714622e5f99d70d96cced0ac4c369c79778a708ac80543d23d652bbaf5ed38de487e796bec538
6
+ metadata.gz: f66be1c49609d4c246a552c8d3f92351c84923a7c21d35e5934ad5f91e6949e80e0742f0576f8e16bcb3ab4790e91f581121f5f8e9f028dd725b8d6170f466c7
7
+ data.tar.gz: 11a5039f7a3584bccd7b2e1546014b401a48eb37c6a5e04f713811665026d6abd160eb6d0c36df70e8329d3328b747807b6586b7afe235a08b06ab7e43193427
data/CHANGELOG CHANGED
@@ -1,5 +1,8 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 0.2.1
4
+ fix tremor. introduced selenium with firefox. sorry this is dirty, but I have to. tremor somehow doesn't work with phantomjs
5
+
3
6
  ## 0.1.69
4
7
  fix criteo
5
8
 
data/Gemfile CHANGED
@@ -16,3 +16,4 @@ gem 'rubyzip'
16
16
  gem 'google-api-client', '~> 0.8.2'
17
17
  gem 'cheddar'
18
18
  gem 'roo', '~> 2.4.0'
19
+ gem 'selenium-webdriver'
@@ -2,6 +2,13 @@ require 'date'
2
2
  require_relative 'base_client'
3
3
 
4
4
  class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
5
+ def init_client
6
+ Capybara.register_driver :selenium do |app|
7
+ Capybara::Selenium::Driver.new(app, :browser => :firefox)
8
+ end
9
+ @client = Capybara::Session.new(:selenium)
10
+ end
11
+
5
12
  private
6
13
 
7
14
  def login
@@ -10,8 +17,11 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
10
17
  @client.fill_in 'password', :with => @secret
11
18
  @client.click_button 'Sign In'
12
19
  begin
20
+ retries ||= 0
21
+ sleep 1
13
22
  @client.find :xpath, '//*[text()="REPORTS"]'
14
23
  rescue Exception => e
24
+ retry if (retries += 1) < 10
15
25
  raise e, 'Tremor login error'
16
26
  end
17
27
  end
@@ -30,6 +40,14 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
30
40
  sleep 1
31
41
  @client.find(:xpath, '//*[text()="New"]').click
32
42
  sleep 1
43
+
44
+ # select date
45
+ @client.find(:css, '#customReportsDateRanges').click
46
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
47
+ @client.find(:css, '#timezone').click
48
+ sleep 1
49
+ @client.find_all(:xpath, '//div[text()="Eastern Standard Time"]').first.click
50
+
33
51
  # select group by
34
52
  @client.find(:css, '#availableFieldsListSearch').click
35
53
  @client.find(:xpath, '//*[text()="Supply Domain"]').click
@@ -41,23 +59,25 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
41
59
  @client.find(:xpath, '//*[text()="Completions"]').click
42
60
  @client.find(:xpath, '//*[text()="Total Net Revenue"]').click
43
61
  @client.find(:css, '.custom-report-dropdown-glyph.glyphicon-remove').click
44
- # select date
45
- @client.find(:css, '#customReportsDateRanges').click
46
- @client.find(:xpath, '//*[text()="Yesterday"]').click
62
+ @client.execute_script('window.scrollTo(0,0)')
47
63
  @client.click_button 'Run'
48
64
  sleep 10
65
+ flag_holding = true
49
66
  30.times do |_i| # wait 5 min
50
67
  begin
51
68
  @client.find(:xpath, '//*[text()="Please Hold"]')
52
69
  rescue Exception => e
53
70
  break
71
+ flag_holding = false
54
72
  end
55
73
  sleep 10
56
74
  end
75
+ fail 'Tremor report taking too long. Abort' if flag_holding
57
76
  end
58
77
 
59
78
  def extract_data_from_report
60
- rows = @client.find_all :xpath, '//table/*/tr'
61
- @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
79
+ page = Nokogiri::HTML @client.html
80
+ rows = page.xpath '//table[@id="DataTables_Table_1"]/*/tr'
81
+ @data = rows.map { |tr| tr.css('td,th').map { |td| td.text } }
62
82
  end
63
- end
83
+ end
@@ -1,3 +1,3 @@
1
1
  module AdopsReportScrapper
2
- VERSION = "0.1.69"
2
+ VERSION = "0.2.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: adops_report_scrapper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.69
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stayman Hou
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-03-21 00:00:00.000000000 Z
11
+ date: 2017-03-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: httpclient