adops_report_scrapper 0.1.0

data/lib/adops_report_scrapper/conversant_client.rb

@@ -0,0 +1,66 @@
+require 'date'
+require_relative 'base_client'
+require 'nokogiri'
+
+class AdopsReportScrapper::ConversantClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://admin.valueclickmedia.com/corp/login'
+    @client.fill_in 'user_name', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Submit'
+    begin
+      @client.find :xpath, '//*[contains(text(),"Reports")]'
+    rescue Exception => e
+      raise e, 'Conversant login error'
+    end
+  end
+
+  def scrap
+    @client.click_link 'Earnings'
+
+    sites = @client.find_all(:css, '#search-site_id > option')
+    n_sites = []
+    sites.each do |site|
+      site_id = site[:value].to_i
+      next if site_id <= 0
+      site_name = site.text
+      n_sites << { site_id: site_id, site_name: site_name }
+    end
+
+    @data = [['Site', 'Device', 'Country', 'Impressions', 'Clicks', 'Earnings']]
+    n_sites.each do |site|
+      extract_data site
+    end
+  end
+
+  def extract_data(site)
+    d_us_datum = get_line_data(site, :desktop, :us)
+    m_us_datum = get_line_data(site, :mobile, :us)
+    d_all_datum = get_line_data(site, :desktop, :all)
+    m_all_datum = get_line_data(site, :mobile, :all)
+    d_intl_datum = d_all_datum.zip(d_us_datum).map { |x, y| x - y }
+    m_intl_datum = m_all_datum.zip(m_us_datum).map { |x, y| x - y }
+
+    d_us_datum = [site[:site_name], 'Desktop', 'US'] + d_us_datum
+    m_us_datum = [site[:site_name], 'Mobile', 'US'] + m_us_datum
+    d_intl_datum = [site[:site_name], 'Desktop', 'Intl'] + d_intl_datum
+    m_intl_datum = [site[:site_name], 'Mobile', 'Intl'] + m_intl_datum
+
+    [d_us_datum, m_us_datum, d_intl_datum, m_intl_datum].each do |datum|
+      next if datum[2..-1] == [0,0,0]
+      @data << datum
+    end
+  end
+
+  def get_line_data(site, device, country)
+    date_str = @date.strftime '%Y-%m-%d'
+    supply_type_id_map = { desktop: 1, mobile: 2 }
+    country_id_map = { all: 0, us: 254 }
+    @client.visit "https://pub.valueclickmedia.com/reports/earnings/detailed_media_grid?start_date=#{date_str}&end_date=#{date_str}&site_id=#{site[:site_id]}&media_type_id=&supply_type_id=#{supply_type_id_map[device]}&country_id=#{country_id_map[country]}"
+    doc = Nokogiri::XML(@client.body)
+    cells = doc.css('cell')
+    return [cells[1].content.to_i, cells[2].content.to_i, cells[-1].content.to_f] # [imp, click, earning]
+  end
+end

data/lib/adops_report_scrapper/criteo_client.rb

@@ -0,0 +1,25 @@
+require 'date'
+require_relative 'base_client'
+require 'httpclient'
+
+class AdopsReportScrapper::CriteoClient < AdopsReportScrapper::BaseClient
+  private
+
+  def init_client
+  end
+
+  def before_quit_with_error
+  end
+
+  def scrap
+    @data = []
+    date_str = @date.strftime('%Y-%m-%d')
+
+    response = HTTPClient.get "https://publishers.criteo.com/api/2.0/stats.json", apitoken: @secret, begindate: date_str, enddate: date_str
+
+    data = JSON.parse response.body
+    header = data[0].keys
+    @data = [header]
+    @data += data.map { |datum| header.map { |key| datum[key].is_a?(Hash) ? datum[key]['value'] : datum[key] } }
+  end
+end

data/lib/adops_report_scrapper/facebookaudience_client.rb

@@ -0,0 +1,56 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::FacebookaudienceClient < AdopsReportScrapper::BaseClient
+  private
+
+  def init_client
+    fail 'please specify facebook app id' unless @options['app_id']
+    @app_id = @options['app_id']
+    super
+  end
+
+  def login
+    @client.visit "https://developers.facebook.com/apps/#{@app_id}/audience-network/placement"
+    @client.fill_in 'email', :with => @login
+    @client.fill_in 'pass', :with => @secret
+    @client.click_button 'Log In'
+    begin
+      @client.find :xpath, '//*[text()="Dashboard"]'
+    rescue Exception => e
+      raise e, 'Facebookaudience login error'
+    end
+  end
+
+  def scrap
+    @client.find(:xpath, '//*[text()="All Ad Placements"]').click
+    placements = @client.find_all(:xpath, '//span[../../a[@role="menuitem"]]')
+    placements = placements.map(&:text)
+    @client.find(:xpath, '//*[text()="All Ad Placements"]').click
+    @prev_placement = 'All Ad Placements'
+    @data = []
+    placements.each do |placement|
+      request_report placement
+      extract_data_from_report placement
+    end
+  end
+
+  def request_report(placement)
+    @client.find(:xpath, "//*[text()=\"#{@prev_placement}\"]").click
+    @client.find(:xpath, "//*[text()=\"#{placement}\"]").click
+    @prev_placement = placement.match(/^(.+) \(\d+\)/).captures[0]
+    sleep 1
+  end
+
+  def extract_data_from_report(placement)
+    if @data.count == 0
+      header = @client.find :xpath, '//table/thead/tr'
+      @data << ['Placement'] + header.find_css('td,th').map { |td| td.visible_text }
+    end
+    data_str = @date.strftime '%a %b %d, %Y'
+    rows = @client.find_all :xpath, "//table/*/tr[./td[text()=\"#{data_str}\"]]"
+    return if rows.count == 0
+    row = rows.first
+    @data << [placement] + row.find_css('td,th').map { |td| td.visible_text }
+  end
+end

data/lib/adops_report_scrapper/gcs_client.rb

@@ -0,0 +1,56 @@
+require 'date'
+require_relative 'base_client'
+
+# gcs sometimes doesn't update data in 24 hours
+
+class AdopsReportScrapper::GcsClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://www.google.com/insights/consumersurveys/your-surveys'
+    @client.fill_in 'Email', :with => @login
+    @client.click_button 'Next'
+    @client.fill_in 'Passwd', :with => @secret
+    @client.click_button 'Sign in'
+    begin
+      @client.find :xpath, '//*[text()="Sites"]'
+    rescue Exception => e
+      raise e, 'Gcs login error'
+    end
+  end
+
+  def scrap
+    sites = @client.find_all(:xpath, '//*[contains(@class,"sites-menu-item")]', visible: false)
+    sites = sites.to_a
+    sites.pop
+    n_sites = sites.map do |site|
+      {
+        name: site.text(:all),
+        url: site[:href].sub('settings', 'report')
+      }
+    end
+    @data = []
+    n_sites.each do |site|
+      request_report site
+      extract_data_from_report site
+    end
+  end
+
+  def request_report(site)
+    @client.visit site[:url]
+  end
+
+  def extract_data_from_report(site)
+    rows = @client.find_all :xpath, %Q{//table/*/tr[./td[contains(text(),"#{@date.strftime('%b')}") and contains(text(),"#{@date.strftime('%e, %Y')}")]]}
+    return if rows.count == 0
+    if @data.count == 0
+      header = @client.find :xpath, '//table/thead/tr'
+      n_header = header.find_css('td,th').map { |td| td.visible_text }
+      n_header.unshift 'Site'
+      @data << n_header
+    end
+    row = rows[0].find_css('td,th').map { |td| td.visible_text }
+    row.unshift site[:name]
+    @data << row
+  end
+end

data/lib/adops_report_scrapper/liveintent_client.rb

@@ -0,0 +1,90 @@
+require 'date'
+require_relative 'base_client'
+require 'httpclient'
+require 'roo'
+
+class AdopsReportScrapper::LiveintentClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://lfm.liveintent.com/'
+    @client.fill_in 'username', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Login'
+    begin
+      @client.find :xpath, '//*[text()="Analysis"]'
+    rescue Exception => e
+      raise e, 'Liveintent login error'
+    end
+  end
+
+  def scrap
+    request_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[text()="Analysis"]').click
+    @client.find(:xpath, '//*[contains(text(),"Reporting")]').click
+    @client.find(:xpath, '//*[text()="New"]').click
+    sleep 1
+    @client.find(:xpath, '//*[text()="Publisher ID"]').click
+    @client.find(:xpath, '//*[text()="Ad Slot ID"]').click
+    @client.find(:xpath, '//*[text()="Add additional split"]').click
+    @client.find(:xpath, '//*[text()="Year/Month"]').click
+    @client.find(:xpath, '//*[text()="Device Type (inexact values)"]').click
+    @client.find(:xpath, '//input[@id="intervalBegin"]').set @date.strftime('%Y-%m-%d / %Y-%m-%d')
+    @client.click_button 'Download'
+    wait_for_spin
+
+    request_data = @client.driver.network_traffic.last.instance_variable_get(:@data)
+    report_file_url = @client.driver.network_traffic.last.url
+
+    cookies = @client.driver.cookies
+    @client = HTTPClient.new
+    @client.cookie_manager.cookies = cookies.values.map do |cookie|
+      cookie = cookie.instance_variable_get(:@attributes)
+      HTTP::Cookie.new cookie
+    end
+
+    header = {
+      Accept: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      Referer: 'https://lfm.liveintent.com/reporting/',
+      Origin: 'https://lfm.liveintent.com',
+      'User-Agent': 'Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1',
+      'Content-Type': 'application/json',
+    }
+
+    @client.receive_timeout = 300
+    response = @client.post(report_file_url, header: header, body: request_data['postData'] )
+
+    tmpfile = Tempfile.new('liveintent.xlsx')
+    begin
+      tmpfile.binmode
+      tmpfile.write(response.body)
+      tmpfile.close
+
+      xlsx = Roo::Spreadsheet.open(tmpfile.path, extension: :xlsx)
+      extract_data_from_report xlsx
+    ensure
+      tmpfile.close
+      tmpfile.unlink   # deletes the temp file
+    end
+
+  end
+
+  def extract_data_from_report(xlsx)
+    @data = xlsx.to_a.reject { |row| row[1] == '(totals)' || row[0] == '(totals)' }
+  end
+
+  def wait_for_spin
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '.fa.fa-spinner.fa-spin')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 5
+  end
+end

data/lib/adops_report_scrapper/marfeel_client.rb

@@ -0,0 +1,59 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::MarfeelClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://insight.marfeel.com/hub/login'
+    @client.fill_in 'j_username', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Login'
+    begin
+      @client.find :xpath, '//*[text()="REPORTING"]'
+    rescue Exception => e
+      raise e, 'Marfeel login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.visit 'https://insight.marfeel.com/hub/insight/reporting?date=1d'
+    sleep 1
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table/*/tr'
+    n_rows = rows.map do |tr|
+      tr.find_css('td,th').map do |td|
+        datum = td.visible_text
+        if datum.end_with? ' K'
+          datum = datum.to_f * 1_000
+        elsif datum.end_with? ' M'
+          datum = datum.to_f * 1_000_000
+        end
+        datum.to_s
+      end
+    end
+    n_rows[0][0] = 'Site'
+    site = nil
+    l = n_rows[0].count
+    @data = n_rows.map do |row|
+      if row.count == l
+        row[0] = row[0].split(' $')[0]
+        site = row[0]
+      else
+        row.unshift site
+      end
+      row
+    end
+    (@data.count - 1).times do |i|
+      i.even? ? @data[i+1].unshift('Smartphone') : @data[i+1].unshift('Tablet')
+    end
+    @data[0].unshift 'Device'
+  end
+end

data/lib/adops_report_scrapper/nativo_client.rb

@@ -0,0 +1,60 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::NativoClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://admin.nativo.net/'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Log In'
+    begin
+      @client.find :xpath, '//*[text()="Reports"]'
+    rescue Exception => e
+      raise e, 'Nativo login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.click_link 'Reports'
+    @client.click_link 'Marketplace Campaigns'
+
+    sleep 1
+    @client.find(:css, '.date-range').click
+    sleep 1
+    @client.find(:xpath, '//*[text()="Yesterday"]').click
+
+    @client.check 'Publisher'
+    @client.check 'Device'
+
+    @client.check 'Clicks'
+    @client.check 'Publisher Revenue'
+    @client.check 'Video Views'
+    @client.check 'Video Views to 100%'
+    sleep 1
+    wait_for_loading
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//div/table/*[self::thead|self::tbody]/tr'
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+
+  def wait_for_loading
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:css, '.loading')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 1
+  end
+end

data/lib/adops_report_scrapper/netseer_client.rb

@@ -0,0 +1,73 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::NetseerClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://publisher.netseer.com/login'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Login'
+    begin
+      @client.find :css, '.icon-calender'
+    rescue Exception => e
+      raise e, 'Netseer login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    sleep 3
+    @client.find(:xpath, '//*[contains(text(),"Run Reports")]').click
+    wait_for_loading
+
+    @client.find(:css, '.icon-calender').click
+    @client.fill_in 'daterangepicker_start', :with => @date.strftime('%m/%d/%Y')
+    @client.fill_in 'daterangepicker_end', :with => @date.strftime('%m/%d/%Y')
+    @client.click_button 'Apply'
+    sleep 3
+
+    @client.choose 'Day'
+    @client.choose 'Tag'
+    @client.check 'Delivery Medium'
+
+    @client.click_link_or_button 'Run Report'
+    wait_for_loading
+
+  end
+
+  def extract_data_from_report
+    @data = []
+    loop do
+      rows = @client.find_all :xpath, '//table/*/tr'
+      rows = rows.to_a
+      header = rows.shift
+      if @data.count == 0
+        n_header = header.find_css('td,th').map { |td| td.visible_text }
+        @data << n_header
+      end
+      @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+      pagee = @client.find(:xpath, '//*[contains(text(),"Showing ")]').text.match(/to (\d+) of (\d+)/).captures
+      break if pagee[0] == pagee[1]
+      @client.find(:css, 'a > .fa.fa-angle-right').click
+      wait_for_loading
+    end
+  end
+
+  def wait_for_loading
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:xpath, '//*[text()="Loading ..."]')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 5
+  end
+end