adops_report_scrapper 0.1.0

data/lib/adops_report_scrapper/adforge_client.rb

@@ -0,0 +1,35 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::AdforgeClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://495.as.adforgeinc.com/www/admin/index.php'
+    @client.fill_in 'username', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Login'
+    begin
+      @client.find :xpath, '//*[text()="Reports"]'
+    rescue Exception => e
+      raise e, 'Adforge login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//option[text()="Yesterday"]').select_option
+    sleep 5
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//td/table/*/tr'
+    rows = rows.to_a
+    rows.delete_at 1
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+end

data/lib/adops_report_scrapper/adiply_client.rb

@@ -0,0 +1,49 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::AdiplyClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://pub.adiply.com/login'
+    @client.fill_in '_username', :with => @login
+    @client.fill_in '_password', :with => @secret
+    @client.click_button 'Sign me in'
+    begin
+      @client.find :xpath, '//*[text()="Go"]'
+    rescue Exception => e
+      raise e, 'Adiply login error'
+    end
+  end
+
+  def scrap
+    zones = @client.find_all(:css, '#AppBundle_filtersForm_zone > option')
+    zones = zones.to_a
+    zones.shift
+    zones = zones.map { |zone| zone.text }
+    @client.find(:xpath, '//a[contains(text(),"Performance")]').click
+    @client.find(:css, '.dr-presets').click
+    @client.find(:xpath, '//*[contains(text(),"Last 30 days")]').click
+    @data = []
+    zones.each do |zone|
+      extract_data zone
+    end
+  end
+
+  def extract_data(zone)
+    @client.find(:xpath, "//option[text()=\"#{zone}\"]").select_option
+    @client.find(:xpath, '//*[text()="Go"]').click
+    sleep 2
+    date_str = @date.strftime '%m/%d/%Y'
+    if @data.count == 0
+      header = @client.find_all(:xpath, '//table/thead/tr/th').map { |th| th.text }
+      header[-1] = 'Zone'
+      @data << header
+    end
+    tds = @client.find_all :xpath, "//td[../td[contains(text(),\"#{date_str}\")]]"
+    row = tds.map { |td| td.text }
+    row[-1] = zone
+    row[0] = date_str
+    @data << row
+  end
+end

data/lib/adops_report_scrapper/adsense_client.rb

@@ -0,0 +1,55 @@
+require 'date'
+require_relative 'base_client'
+require 'google/api_client'
+require 'google/api_client/service'
+require 'google/api_client/client_secrets'
+require 'google/api_client/auth/file_storage'
+require 'google/api_client/auth/installed_app'
+require 'csv'
+
+# require option with network_id to be passed into constructor
+class AdopsReportScrapper::AdsenseClient < AdopsReportScrapper::BaseClient
+  API_NAME = 'adsense'
+  API_VERSION = 'v1.4'
+  CREDENTIAL_STORE_FILE = "#{API_NAME}-oauth2.json"
+  API_SCOPE = 'https://www.googleapis.com/auth/adsense.readonly'
+
+  private
+
+  def init_client
+    fail 'please specify adsense account id' unless @options['account_id']
+    @account_id = @options['account_id']
+    authorization = nil
+
+    file_storage = Google::APIClient::FileStorage.new(CREDENTIAL_STORE_FILE)
+    if file_storage.authorization.nil?
+      flow = Google::APIClient::InstalledAppFlow.new(
+        :client_id => @login,
+        :client_secret => @secret,
+        :scope => [API_SCOPE]
+      )
+      authorization = flow.authorize(file_storage)
+    else
+      authorization = file_storage.authorization
+    end
+
+    @client = Google::APIClient::Service.new(API_NAME, API_VERSION,
+      {
+        :application_name => "Ruby #{API_NAME} ad report scrapper",
+        :application_version => '1.0.0',
+        :authorization => authorization
+      }
+    )
+  end
+
+  def scrap
+    result = @client.accounts.reports.generate(
+        :accountId => @account_id,
+        :startDate => 'today-1d',
+        :endDate => 'today-1d',
+        :metric => ['AD_REQUESTS', 'INDIVIDUAL_AD_IMPRESSIONS', 'CLICKS', 'EARNINGS'],
+        :dimension => ['DATE', 'AD_UNIT_CODE', 'AD_UNIT_NAME', 'COUNTRY_CODE', 'PLATFORM_TYPE_NAME'],
+        :alt => 'csv').execute
+    @data = CSV.parse(result.body)
+  end
+end

data/lib/adops_report_scrapper/adsupply_client.rb

@@ -0,0 +1,27 @@
+require 'date'
+require_relative 'base_client'
+require 'rest-client'
+
+class AdopsReportScrapper::AdsupplyClient < AdopsReportScrapper::BaseClient
+  def init_client
+  end
+
+  def before_quit_with_error
+  end
+
+  private
+
+  def scrap
+    date_str = @date.strftime('%-m/%-d/%Y')
+    time_zone_id = 'Eastern Standard Time'
+
+    response = RestClient.post "https://ui.adsupply.com/PublicPortal/Publisher/#{@login}/Report/Export", SqlCommandId: '', ExportToExcel: 'False', IsOLAP: 'False', DateFilter: date_str, TimeZoneId: time_zone_id, Grouping: '1', 'DimPublisher.Value': "#{@login}~", 'DimPublisher.IsActive': 'True', 'DimSiteName.Value': '', 'DimSiteName.IsActive': 'True', 'DimCountry.Value': '', 'DimCountry.IsActive': 'True', 'DimMediaType.Value': '', 'DimMediaType.IsActive': 'True', ApiKey: @secret
+
+    data = JSON.parse response
+    header = data[0].keys
+    @data = [header]
+    @data += data.map do |datum|
+      header.map { |key| datum[key] }
+    end
+  end
+end

data/lib/adops_report_scrapper/adx_client.rb

@@ -0,0 +1,55 @@
+require 'date'
+require_relative 'base_client'
+require 'google/api_client'
+require 'google/api_client/service'
+require 'google/api_client/client_secrets'
+require 'google/api_client/auth/file_storage'
+require 'google/api_client/auth/installed_app'
+require 'csv'
+
+# require option with network_id to be passed into constructor
+class AdopsReportScrapper::AdxClient < AdopsReportScrapper::BaseClient
+  API_NAME = 'adexchangeseller'
+  API_VERSION = 'v2.0'
+  CREDENTIAL_STORE_FILE = "#{API_NAME}-oauth2.json"
+  API_SCOPE = 'https://www.googleapis.com/auth/adexchange.seller.readonly'
+
+  private
+
+  def init_client
+    fail 'please specify adx account id' unless @options['account_id']
+    @account_id = @options['account_id']
+    authorization = nil
+
+    file_storage = Google::APIClient::FileStorage.new(CREDENTIAL_STORE_FILE)
+    if file_storage.authorization.nil?
+      flow = Google::APIClient::InstalledAppFlow.new(
+        :client_id => @login,
+        :client_secret => @secret,
+        :scope => [API_SCOPE]
+      )
+      authorization = flow.authorize(file_storage)
+    else
+      authorization = file_storage.authorization
+    end
+
+    @client = Google::APIClient::Service.new(API_NAME, API_VERSION,
+      {
+        :application_name => "Ruby #{API_NAME} ad report scrapper",
+        :application_version => '1.0.0',
+        :authorization => authorization
+      }
+    )
+  end
+
+  def scrap
+    result = @client.accounts.reports.generate(
+        :accountId => @account_id,
+        :startDate => 'today-1d',
+        :endDate => 'today-1d',
+        :metric => ['AD_REQUESTS', 'AD_IMPRESSIONS', 'CLICKS', 'EARNINGS'],
+        :dimension => ['DATE', 'DFP_AD_UNITS', 'DFP_AD_UNIT_ID', 'COUNTRY_CODE', 'PLATFORM_TYPE_NAME'],
+        :alt => 'csv').execute
+    @data = CSV.parse(result.body)
+  end
+end

data/lib/adops_report_scrapper/base_client.rb

@@ -0,0 +1,74 @@
+require 'date'
+require 'phantomjs'
+require 'capybara'
+require 'capybara/poltergeist'
+
+class AdopsReportScrapper::BaseClient
+  # login: username, id, email, or api token
+  # secret: password or api secret
+  # options: {
+  #   :date => (optional) default: yesterday
+  # }
+  def initialize(login, secret, options = nil)
+    @login = login
+    @secret = secret
+    @options = options || {}
+    @date = @options[:date] || Date.today.prev_day
+  end
+
+  # date: (optional)
+  # return data in array of array, first array is the headers, no total included
+  def get_data(date = nil)
+    @date = date if date
+    fail "specified date is not supported by this scrapper #{self.class.name}" unless date_supported?
+    init_client
+    login
+    begin
+      scrap
+    rescue Exception => e
+      begin
+        before_quit_with_error
+        logout
+      rescue Exception => _e
+        # do nothing
+      end
+      raise e
+    end
+    logout
+    return @data
+  end
+
+  def init_client
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app, :phantomjs => Phantomjs.path)
+    end
+    Capybara.default_max_wait_time = 10
+    @client = Capybara::Session.new(:poltergeist)
+    @client.driver.browser.js_errors = false
+    @client.driver.resize(1920, 985)
+  end
+
+  def login
+    # do nothing by default
+  end
+
+  def scrap
+    # do nothing by default
+  end
+
+  # logout can be optional
+  def logout
+    # do nothing by default
+  end
+
+  def before_quit_with_error
+    @client.save_screenshot
+  end
+
+  # by default only support yesterday
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date == Date.today.prev_day
+    false
+  end
+end

data/lib/adops_report_scrapper/brightroll_client.rb

@@ -0,0 +1,47 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::BrightrollClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://login.brightroll.com/login'
+    @client.fill_in 'user_login', :with => @login
+    @client.fill_in 'user_password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[text()="Tags"]'
+    rescue Exception => e
+      raise e, 'Brightroll login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[text()="Tags"]').click
+    @client.find(:css, '.details-date-filter').click
+    # select date
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '.bubble-loader.bubble-loader-3')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 10
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table[1]/*/tr'
+    rows = rows.to_a
+    rows.delete_at 1
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+end

data/lib/adops_report_scrapper/browsi_client.rb

@@ -0,0 +1,44 @@
+require 'date'
+require_relative 'base_client'
+require 'open-uri'
+
+class AdopsReportScrapper::BrowsiClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://reports.brow.si'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.find(:xpath, '//*[text()="Login"]').click
+    begin
+      @client.find :css, '.ico-calendar'
+    rescue Exception => e
+      raise e, 'Browsi login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    date_range_obj = {
+      startDate: @date.to_time.utc.strftime('%FT%T.000Z'),
+      endDate: (@date.to_time+86400-1).utc.strftime('%FT%T.999Z')
+    }
+    @client.visit "https://reports.brow.si/client/app/index.html#/report/home?dateRange=#{URI::encode(date_range_obj.to_json)}"
+    sleep 10
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//li[@ng-class="{opened:site.isGraphOpen}"]'
+    @data = [['Site', 'Revenues', 'Page Views']]
+    @data += rows.map do |row|
+      site = row.find_css('.header-full').first.visible_text
+      rev = @client.find_all(:xpath, row.path+'//*[../span[text()="Revenues"]]').first.text
+      pv = @client.find_all(:xpath, row.path+'//*[../span[text()="Page Views"]]').first.text
+      [site, rev, pv]
+    end
+  end
+end

data/lib/adops_report_scrapper/contentad_client.rb

@@ -0,0 +1,91 @@
+require 'date'
+require_relative 'base_client'
+
+# please ensure that all live widgets have distinct name
+
+class AdopsReportScrapper::ContentadClient < AdopsReportScrapper::BaseClient
+  def init_client
+    Capybara.register_driver :poltergeist do |app|
+      Capybara::Poltergeist::Driver.new(app, :phantomjs => Phantomjs.path)
+    end
+    Capybara.default_max_wait_time = 3
+    @client = Capybara::Session.new(:poltergeist)
+    @client.driver.browser.js_errors = false
+    @client.driver.resize(1920, 985)
+  end
+
+  private
+
+  def login
+    @client.visit 'https://www.content.ad/Login.aspx'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Log In'
+    begin
+      @client.find :xpath, '//*[text()="Widget Report"]'
+    rescue Exception => e
+      raise e, 'Contentad login error'
+    end
+  end
+
+  def scrap
+    @client.find(:xpath, '//option[text()="Yesterday"]').select_option
+    @client.click_link 'Apply'
+    wait_for_loading
+
+    @client.find(:xpath, '//span[text()="Widgets"]').click
+    wait_for_loading
+    
+    rows = @client.find_all :xpath, '//table/tbody/tr'
+    widgets = rows.map do |row|
+      row = row.find_css('td').map { |td| td.visible_text }
+      { widget_name: row[0], domain_name: row[1] }
+    end
+
+    @client.visit 'https://app.content.ad/Publisher/DeviceGeoReport'
+    @client.find(:xpath, '//option[text()="Yesterday"]').select_option
+    @client.click_link 'Apply'
+    wait_for_loading
+
+    @data = []
+    widgets.each do |widget|
+      request_report widget
+    end
+  end
+
+  def request_report(widget)
+    widget_options = @client.find(:xpath, "//option[text()=\"#{widget[:widget_name]}\"]").select_option
+    @client.click_link 'Apply'
+    wait_for_loading
+    extract_data_from_report widget
+  end
+
+  def extract_data_from_report(widget)
+    rows = @client.find_all :xpath, '//table/*/tr'
+    rows = rows.to_a
+    header = rows.shift
+    if @data.count == 0
+      header = ['Date', 'Domain', 'Widget'] + header.find_css('td,th').map { |td| td.visible_text }
+      @data << header
+    end
+    rows.pop
+    @data += rows.map do |tr|
+      row = tr.find_css('td,th').map do |td|
+        td.visible_text
+      end
+      [@date.to_s, widget[:domain_name], widget[:widget_name]] + row
+    end
+  end
+
+  def wait_for_loading
+    18.times do |_i| # wait 3 min
+      begin
+        @client.find(:css, '#loadingProgress')
+      rescue Exception => e
+        break
+      end
+      sleep 3
+    end
+    sleep 1
+  end
+end