adops_report_scrapper 0.1.69 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b57c737ccfc9f5d880ce391d9f355dedaf8bbc70
4
- data.tar.gz: a5f2495fa1de0149097f909691a285aa14720bdc
3
+ metadata.gz: ddb35e06ab9f273a789732d953f60131dc470ad9
4
+ data.tar.gz: 3f986c6d5bf899335bcfa705ec056786b8a5859e
5
5
  SHA512:
6
- metadata.gz: cd3b2a29fb54892ee2dfbec9731ee2c5cc72b9eeb4c54d82d15b2972427761e0504ba4be9e1cf49a2c186bb086b1ffad3b4102da97a42bce01c54a638f75fadf
7
- data.tar.gz: 822e3682bafe46d6b90b2b9a5f73094245d5f1c877890134a7d714622e5f99d70d96cced0ac4c369c79778a708ac80543d23d652bbaf5ed38de487e796bec538
6
+ metadata.gz: f66be1c49609d4c246a552c8d3f92351c84923a7c21d35e5934ad5f91e6949e80e0742f0576f8e16bcb3ab4790e91f581121f5f8e9f028dd725b8d6170f466c7
7
+ data.tar.gz: 11a5039f7a3584bccd7b2e1546014b401a48eb37c6a5e04f713811665026d6abd160eb6d0c36df70e8329d3328b747807b6586b7afe235a08b06ab7e43193427
data/CHANGELOG CHANGED
@@ -1,5 +1,8 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 0.2.1
4
+ fix tremor. introduced selenium with firefox. sorry this is dirty, but I have to. tremor somehow doesn't work with phantomjs
5
+
3
6
  ## 0.1.69
4
7
  fix criteo
5
8
 
data/Gemfile CHANGED
@@ -16,3 +16,4 @@ gem 'rubyzip'
16
16
  gem 'google-api-client', '~> 0.8.2'
17
17
  gem 'cheddar'
18
18
  gem 'roo', '~> 2.4.0'
19
+ gem 'selenium-webdriver'
@@ -2,6 +2,13 @@ require 'date'
2
2
  require_relative 'base_client'
3
3
 
4
4
  class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
5
+ def init_client
6
+ Capybara.register_driver :selenium do |app|
7
+ Capybara::Selenium::Driver.new(app, :browser => :firefox)
8
+ end
9
+ @client = Capybara::Session.new(:selenium)
10
+ end
11
+
5
12
  private
6
13
 
7
14
  def login
@@ -10,8 +17,11 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
10
17
  @client.fill_in 'password', :with => @secret
11
18
  @client.click_button 'Sign In'
12
19
  begin
20
+ retries ||= 0
21
+ sleep 1
13
22
  @client.find :xpath, '//*[text()="REPORTS"]'
14
23
  rescue Exception => e
24
+ retry if (retries += 1) < 10
15
25
  raise e, 'Tremor login error'
16
26
  end
17
27
  end
@@ -30,6 +40,14 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
30
40
  sleep 1
31
41
  @client.find(:xpath, '//*[text()="New"]').click
32
42
  sleep 1
43
+
44
+ # select date
45
+ @client.find(:css, '#customReportsDateRanges').click
46
+ @client.find(:xpath, '//*[text()="Yesterday"]').click
47
+ @client.find(:css, '#timezone').click
48
+ sleep 1
49
+ @client.find_all(:xpath, '//div[text()="Eastern Standard Time"]').first.click
50
+
33
51
  # select group by
34
52
  @client.find(:css, '#availableFieldsListSearch').click
35
53
  @client.find(:xpath, '//*[text()="Supply Domain"]').click
@@ -41,23 +59,25 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
41
59
  @client.find(:xpath, '//*[text()="Completions"]').click
42
60
  @client.find(:xpath, '//*[text()="Total Net Revenue"]').click
43
61
  @client.find(:css, '.custom-report-dropdown-glyph.glyphicon-remove').click
44
- # select date
45
- @client.find(:css, '#customReportsDateRanges').click
46
- @client.find(:xpath, '//*[text()="Yesterday"]').click
62
+ @client.execute_script('window.scrollTo(0,0)')
47
63
  @client.click_button 'Run'
48
64
  sleep 10
65
+ flag_holding = true
49
66
  30.times do |_i| # wait 5 min
50
67
  begin
51
68
  @client.find(:xpath, '//*[text()="Please Hold"]')
52
69
  rescue Exception => e
53
70
  break
71
+ flag_holding = false
54
72
  end
55
73
  sleep 10
56
74
  end
75
+ fail 'Tremor report taking too long. Abort' if flag_holding
57
76
  end
58
77
 
59
78
  def extract_data_from_report
60
- rows = @client.find_all :xpath, '//table/*/tr'
61
- @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
79
+ page = Nokogiri::HTML @client.html
80
+ rows = page.xpath '//table[@id="DataTables_Table_1"]/*/tr'
81
+ @data = rows.map { |tr| tr.css('td,th').map { |td| td.text } }
62
82
  end
63
- end
83
+ end
@@ -1,3 +1,3 @@
1
1
  module AdopsReportScrapper
2
- VERSION = "0.1.69"
2
+ VERSION = "0.2.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: adops_report_scrapper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.69
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stayman Hou
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-03-21 00:00:00.000000000 Z
11
+ date: 2017-03-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: httpclient