adops_report_scrapper 0.1.69 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b57c737ccfc9f5d880ce391d9f355dedaf8bbc70
-  data.tar.gz: a5f2495fa1de0149097f909691a285aa14720bdc
+  metadata.gz: ddb35e06ab9f273a789732d953f60131dc470ad9
+  data.tar.gz: 3f986c6d5bf899335bcfa705ec056786b8a5859e
 SHA512:
-  metadata.gz: cd3b2a29fb54892ee2dfbec9731ee2c5cc72b9eeb4c54d82d15b2972427761e0504ba4be9e1cf49a2c186bb086b1ffad3b4102da97a42bce01c54a638f75fadf
-  data.tar.gz: 822e3682bafe46d6b90b2b9a5f73094245d5f1c877890134a7d714622e5f99d70d96cced0ac4c369c79778a708ac80543d23d652bbaf5ed38de487e796bec538
+  metadata.gz: f66be1c49609d4c246a552c8d3f92351c84923a7c21d35e5934ad5f91e6949e80e0742f0576f8e16bcb3ab4790e91f581121f5f8e9f028dd725b8d6170f466c7
+  data.tar.gz: 11a5039f7a3584bccd7b2e1546014b401a48eb37c6a5e04f713811665026d6abd160eb6d0c36df70e8329d3328b747807b6586b7afe235a08b06ab7e43193427

data/CHANGELOG

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 0.2.1
+fix tremor. introduced selenium with firefox. sorry this is dirty, but I have to. tremor somehow doesn't work with phantomjs
+
 ## 0.1.69
 fix criteo
 

data/Gemfile

@@ -16,3 +16,4 @@ gem 'rubyzip'
 gem 'google-api-client', '~> 0.8.2'
 gem 'cheddar'
 gem 'roo', '~> 2.4.0'
+gem 'selenium-webdriver'

data/lib/adops_report_scrapper/tremor_client.rb

@@ -2,6 +2,13 @@ require 'date'
 require_relative 'base_client'
 
 class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
+  def init_client
+    Capybara.register_driver :selenium do |app|
+      Capybara::Selenium::Driver.new(app, :browser => :firefox)
+    end
+    @client = Capybara::Session.new(:selenium)
+  end
+
   private
 
   def login
@@ -10,8 +17,11 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
     @client.fill_in 'password', :with => @secret
     @client.click_button 'Sign In'
     begin
+      retries ||= 0
+      sleep 1
       @client.find :xpath, '//*[text()="REPORTS"]'
     rescue Exception => e
+      retry if (retries += 1) < 10
       raise e, 'Tremor login error'
     end
   end
@@ -30,6 +40,14 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
     sleep 1
     @client.find(:xpath, '//*[text()="New"]').click
     sleep 1
+
+    # select date
+    @client.find(:css, '#customReportsDateRanges').click
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+    @client.find(:css, '#timezone').click
+    sleep 1
+    @client.find_all(:xpath, '//div[text()="Eastern Standard Time"]').first.click
+
     # select group by
     @client.find(:css, '#availableFieldsListSearch').click
     @client.find(:xpath, '//*[text()="Supply Domain"]').click
@@ -41,23 +59,25 @@ class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
     @client.find(:xpath, '//*[text()="Completions"]').click
     @client.find(:xpath, '//*[text()="Total Net Revenue"]').click
     @client.find(:css, '.custom-report-dropdown-glyph.glyphicon-remove').click
-    # select date
-    @client.find(:css, '#customReportsDateRanges').click
-    @client.find(:xpath, '//*[text()="Yesterday"]').click
+    @client.execute_script('window.scrollTo(0,0)')
     @client.click_button 'Run'
     sleep 10
+    flag_holding = true
     30.times do |_i| # wait 5 min
       begin
         @client.find(:xpath, '//*[text()="Please Hold"]')
       rescue Exception => e
         break
+        flag_holding = false
       end
       sleep 10
     end
+    fail 'Tremor report taking too long. Abort' if flag_holding
   end
 
   def extract_data_from_report
-    rows = @client.find_all :xpath, '//table/*/tr'
-    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+    page = Nokogiri::HTML @client.html
+    rows = page.xpath '//table[@id="DataTables_Table_1"]/*/tr'
+    @data = rows.map { |tr| tr.css('td,th').map { |td| td.text } }
   end
-end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.69"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.69
+  version: 0.2.1
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-21 00:00:00.000000000 Z
+date: 2017-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient