adops_report_scrapper 0.1.0

data/lib/adops_report_scrapper/openx_client.rb

@@ -0,0 +1,161 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::OpenxClient < AdopsReportScrapper::BaseClient
+  REPORT_NAME = 'Ad Server Report for adops_report_scrapper'
+
+  private
+
+  def login
+    @client.driver.resize 1920, 700
+    fail 'please specify openx account prefix' unless @options['account_prefix']
+    @account_prefix = @options['account_prefix']
+    @client.visit "http://#{@account_prefix}.openx.net/"
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Submit'
+    begin
+      @client.find :xpath, '//*[text()="Reports"]'
+    rescue Exception => e
+      raise e, 'Openx login error'
+    end
+  end
+
+  def scrap
+    request_report
+  end
+
+  def request_report
+    @client.visit 'http://cmci-ui.openx.net/#/reports?tab=my_reports'
+    sleep 5
+
+    begin
+      tries ||= 6
+      @client.find(:css, '#report_frame')
+    rescue Exception => e
+      retry unless (tries -= 1).zero?
+    end
+
+    create_report_if_not_exist
+    
+    @client.within_frame @client.find(:css, '#report_frame') do
+      @client.find(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").click
+
+      begin
+        tries ||= 6
+        @client.find(:css, '.myFrame')
+      rescue Exception => e
+        retry unless (tries -= 1).zero?
+      end
+      
+      @client.within_frame @client.find(:css, '.myFrame') do
+        begin
+          tries ||= 18
+          @client.find(:xpath, '//option[text()="500"]')
+        rescue Exception => e
+          retry unless (tries -= 1).zero?
+        end
+        
+        @client.find(:xpath, '//option[text()="500"]').select_option
+        extract_data_from_report
+      end
+    end
+    sleep 5
+  end
+
+  def create_report_if_not_exist
+    @client.within_frame @client.find(:css, '#report_frame') do
+      ready_elem = nil
+      begin
+        tries ||= 6
+        ready_elem = @client.find(:xpath, '//*[text()="Preconfigured Reports"]')
+      rescue Exception => e
+        retry unless (tries -= 1).zero?
+      end
+
+      fail 'openx report page not ready' unless ready_elem
+    
+      if @client.find_all(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").count == 0
+
+        # create report if not exist
+        @client.find(:xpath, '//*[contains(text(),"Create Report")]').click
+        @client.find(:xpath, '//li/*[text()="Ad Server Report"]').click
+
+        begin
+          tries ||= 6
+          @client.find(:css, '.myFrame')
+        rescue Exception => e
+          retry unless (tries -= 1).zero?
+        end
+        
+        @client.within_frame @client.find(:css, '.myFrame') do
+
+          @client.find(:xpath, '//*[text()="Last Seven Days"]').click
+          @client.find(:xpath, '//*[text()="Yesterday"]').click
+          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
+          sleep 2
+
+          @client.find(:xpath, '//*[text()="Ad Unit"]').click
+          @client.find(:xpath, '//img[../../td//*[text()="User"]]').click
+          @client.find(:xpath, '//*[text()="Country"]').click
+          @client.find(:xpath, '//*[text()="Device Category"]').click
+          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
+          sleep 2
+          
+          @client.find(:xpath, '//*[text()="Paid Impressions"]').click
+          @client.find(:xpath, '//*[text()="Impressions Delivered"]').click
+          @client.find(:xpath, '//*[text()="Ad Requests"]').click
+          @client.find(:xpath, '//img[../../td//*[text()="Revenue"]]').click
+          @client.find(:xpath, '//*[text()="Publisher Revenue"]').click
+          @client.find(:xpath, '//img[../../td//*[text()="Clicks"]]').click
+          @client.find(:xpath, '//div[text()="Clicks"]').click
+          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
+          sleep 2
+
+          @client.find(:xpath, '//*[@value="Save"]').trigger('click')
+
+          begin
+            tries ||= 6
+            @client.find(:xpath, '//*[text()="Save Report"]')
+          rescue Exception => e
+            retry unless (tries -= 1).zero?
+          end
+          
+          @client.fill_in 'saveAsReportName', :with => REPORT_NAME
+          @client.find(:xpath, '//*[@value="Save Report"]').trigger('click')
+
+          begin
+            tries ||= 6
+            @client.find(:xpath, '//*[text()="Report Saved"]')
+          rescue Exception => e
+            retry unless (tries -= 1).zero?
+          end
+          
+          @client.find(:xpath, '//*[text()="Back to My Reports"]').trigger('click')
+        end
+        sleep 2
+      end
+    end
+  end
+
+  def extract_data_from_report
+    @data = []
+    loop do
+      18.times do
+        sleep 10
+        break if @client.find_all(:xpath, '//table[@id="table_UniqueReportID"]/*/tr').count > 0
+      end
+      rows = @client.find_all :xpath, '//table[@id="table_UniqueReportID"]/*/tr'
+      rows = rows.to_a
+      header = rows.shift
+      if @data.count == 0
+        n_header = header.find_css('td,th').map { |td| td.visible_text }
+        @data << n_header
+      end
+      @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }.reject { |row| row[0] == 'Total' }
+      pagee = @client.find(:xpath, '//*[contains(text(),"Showing rows")]').text.match(/-(\d+) of (\d+)\./).captures
+      break if pagee[0] == pagee[1]
+      @client.find_all(:css, '#paginationNext').first.trigger('click')
+    end
+  end
+end

data/lib/adops_report_scrapper/revcontent_client.rb

@@ -0,0 +1,51 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::RevcontentClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://www.revcontent.com/login'
+    @client.fill_in 'name', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[contains(text(),"Widgets")]'
+    rescue Exception => e
+      raise e, 'Revcontent login error'
+    end
+  end
+
+  def scrap
+    @client.find(:xpath, '//*[contains(text(),"Widgets")]').click
+    @client.find(:css, '.fa.fa-calendar').click
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+    @data = []
+    %w(desktoplg desktop tablet phone unknown).each do |device|
+      request_report(device)
+      extract_data_from_report(device)
+    end
+  end
+
+  def request_report(device)
+    @client.find(:css, '.fa.fa-desktop').click
+    @client.find(:xpath, '//*[text()="All Devices"]').click
+    @client.find(:css, '.fa.fa-desktop').click
+    @client.check device
+  end
+
+  def extract_data_from_report(device)
+    rows = @client.find_all :xpath, '//table/*/tr'
+    rows = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+    header = rows.shift
+    rows.shift
+    if @data.count == 0
+      header.unshift 'Device'
+      @data << header
+    end
+    rows.each do |row|
+      row.unshift device
+    end
+    @data += rows
+  end
+end

data/lib/adops_report_scrapper/sonobi_client.rb

@@ -0,0 +1,91 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::SonobiClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://jetstream.sonobi.com/welcome/login.php'
+    @client.fill_in 'user name', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Submit'
+    begin
+      @client.find :xpath, '//*[text()="dashboard"]'
+    rescue Exception => e
+      raise e, 'Sonobi login error'
+    end
+  end
+
+  def scrap
+    @data = []
+    request_report(:us)
+    extract_data_from_report(:us)
+    request_report(:intl)
+    extract_data_from_report(:intl)
+  end
+
+  def request_report(country)
+    is_us = country == :us
+    # all sites
+    @client.find(:xpath, '//*[text()="Reports"]').click
+    sleep 2
+    @client.find(:xpath, '//div[@name="_siteid"]').click
+    sleep 2
+    @client.find(:xpath, '//*[text()="Select All"]').click
+
+    # select country
+    @client.find(:xpath, '//*[text()="Add New Filter"]').click
+    @client.find_all(:xpath, '//option[text()="Country"]').last.select_option
+    sleep 2
+    @client.find(:xpath, '//div[@name="_country"]').click
+    sleep 2
+    if is_us
+      @client.find(:xpath, '//*[text()="United States"]').click
+    else
+      @client.find(:xpath, '//*[text()="Select All"]').click
+      @client.find(:xpath, '//*[@class="remove_icon"][../*[text()="United States"]]').click
+    end
+
+    # check group by
+    @client.check 'Date'
+    @client.check 'Placement'
+    @client.check 'Site'
+    @client.check 'Clicks'
+    @client.check 'Views'
+    @client.check 'Unfilled Impressions'
+    @client.check 'Device Type'
+
+    @client.click_button 'Run Report'
+    sleep 1
+    wait_for_spin
+  end
+
+  def extract_data_from_report(country)
+    rows = @client.find_all :xpath, '//*[@class="reports_tab_item_body"]//table/*/tr'
+    rows = rows.to_a
+    header = rows.shift
+    if @data.count == 0
+      n_header = header.find_css('td,th').map { |td| td.visible_text }
+      n_header << 'Country'
+      @data << n_header
+    end
+    rows.shift
+    @data += rows.map do |row|
+      n_row = row.find_css('td,th').map { |td| td.visible_text }
+      n_row << country.to_s.upcase
+      n_row
+    end
+  end
+
+  def wait_for_spin
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '.circle xlarge')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 5
+  end
+end

data/lib/adops_report_scrapper/springserve_client.rb

@@ -0,0 +1,52 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::SpringserveClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://video.springserve.com/'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Log in'
+    begin
+      @client.find :xpath, '//*[contains(text(),"Reporting")]'
+    rescue Exception => e
+      raise e, 'Springserve login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[contains(text(),"Reporting")]').click
+    @client.find(:xpath, '//*[contains(text(),"Create Reports")]').click
+
+    # select date
+    @client.find(:css, '#date_range_chosen').click
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+
+    @client.find(:css, '#dimensions_chosen').click
+    @client.find(:xpath, '//*[text()="Country"]').click
+
+    @client.find(:xpath, '//*[@value="Run Report"]').click
+
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '#spinner_image')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 10
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table[1]/*/tr'
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }.reject { |row| row[0] == 'Total' }
+  end
+end

data/lib/adops_report_scrapper/tremor_client.rb

@@ -0,0 +1,52 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::TremorClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://console.tremorhub.com/ssp'
+    @client.fill_in 'username', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[text()="REPORTS"]'
+    rescue Exception => e
+      raise e, 'Tremor login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[text()="REPORTS"]').click
+    @client.find(:xpath, '//*[text()="Hierarchical"]').click
+    sleep 1
+    # select group by
+    @client.find(:css, '#hierarchicalReportsGroupBys').click
+    @client.find(:xpath, '//*[text()="AdUnit"]').click
+    @client.find(:css, '#hierarchicalReportsGroupBys').click
+    @client.find(:xpath, '//*[text()="Country"]').click
+    # select date
+    @client.find(:css, '#hierarchicalReportsDateRange').click
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+    @client.click_button 'Run'
+    sleep 10
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:xpath, '//*[text()="Please Hold"]')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table/*/tr'
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+end

data/lib/adops_report_scrapper/triplelift_client.rb

@@ -0,0 +1,72 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::TripleliftClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://console.triplelift.com/login'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Sign in'
+    begin
+      @client.find :xpath, '//*[text()="Reporting"]'
+    rescue Exception => e
+      raise e, 'Triplelift login error'
+    end
+  end
+
+  def scrap
+    @date_str = @date.strftime('%B %d, %Y')
+    @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
+    @publishers = @client.find_all(:xpath, '//*[@ng-click="selectPub(pub)"]').to_a.map { |pub_elem| pub_elem.text(:all) }
+    @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
+    @data = []
+    while @publishers.count > 0
+      extract_data(@publishers.shift)
+    end
+  end
+
+  def extract_data(publisher)
+    @client.find(:xpath, '//*[@ng-if="publishers.length > 1"]').click
+    index = -1 - @publishers.count(publisher)
+    sleep 1
+    @client.find_all(:xpath, "//*[text()=\"#{publisher}\"]")[index].click
+    @client.find(:xpath, '//*[text()="Reporting"]').click
+    sleep 2
+
+    return if @client.find_all(:xpath, '//*[text()="No data available for selected date range"]').count > 0
+
+    @client.find(:xpath, '//*[@model="startDate"]//input').set @date_str
+    sleep 1
+    @client.find(:xpath, '//*[@model="endDate"]//input').set @date_str
+    sleep 1
+    @client.find(:xpath, '//button[../../div[contains(text(),"Group by")]]').click
+    @client.find(:xpath, '//*[text()="Date and Placement"]').click
+    wait_for_spin
+
+    return if @client.find_all(:xpath, '//*[text()="No data available for selected date range"]').count > 0
+
+    rows = @client.find_all :xpath, '//table/*/tr'
+    rows = rows.to_a
+    rows.shift
+    header = rows.shift
+    if @data.count == 0
+      n_header = header.find_css('td,th').map { |td| td.visible_text }
+      @data << n_header
+    end
+    @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+
+  def wait_for_spin
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '.spinner')
+      rescue Exception => e
+        break
+      end
+      sleep 5
+    end
+    sleep 2
+  end
+end