adops_report_scrapper 0.1.36 → 0.1.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8620e4abb3756a733a39ce5bf308586038c5a6ae
-  data.tar.gz: 64f4adc907a5e42d7386f78782d8d4c72e72f0d8
+  metadata.gz: 4a308286c8be74d32479abae098b6a0c56fe5822
+  data.tar.gz: b25af761e68edcbeba8f59f044213159595e945e
 SHA512:
-  metadata.gz: b9c25e199d759b37584c8a034d45cfe5e78832946de0f41edd27fbae5a6d7c3a663a95e051b55e7b168314e208025fc254297fbf32e3c387bc617cd6fbf23267
-  data.tar.gz: 2c5333f175b22d9fb0be6e45225bd38a976e47a2d781ee819920bdcdacce216d3e931c2300498a4be8b9c10f8ff5698051dba0f9b6ba02fae5d5a51fa23fab71
+  metadata.gz: 1e3c40e38be7112945dff112f356b382ff73135b103751a27dc9a39994b3c31d8af5ebbed2453eeabdc73cb5cda1d43a2d94f53de286cb79e68d865d514b88c5
+  data.tar.gz: c739c40809f150ba8a02ef5d17448e0205ecc4b708d64581fc36ad2e3d5f6b2f96fb644dcf1f329829900e4b53734768afe18a4a301cba35ac1dd496086cf09d

data/CHANGELOG

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 0.1.37
+implement undertone
+
 ## 0.1.35
 fixed triple lift for their new ui
 

data/Rakefile

@@ -151,6 +151,16 @@ task :littlethings do
   save_as_csv :littlethings, :littlethings
 end
 
+desc 'Collect rubicon data'
+task :rubicon do
+  save_as_csv :rubicon, :rubicon
+end
+
+desc 'Collect undertone data'
+task :undertone do
+  save_as_csv :undertone, :undertone
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper.rb

@@ -41,3 +41,5 @@ require 'adops_report_scrapper/zedo_client'
 require 'adops_report_scrapper/adtomation_client'
 require 'adops_report_scrapper/rhythmone_client'
 require 'adops_report_scrapper/littlethings_client'
+require 'adops_report_scrapper/rubicon_client'
+require 'adops_report_scrapper/undertone_client'

data/lib/adops_report_scrapper/rubicon_client.rb

@@ -0,0 +1,43 @@
+require 'date'
+require_relative 'base_client'
+require 'rest-client'
+
+class AdopsReportScrapper::RubiconClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date < Date.today
+    false
+  end
+
+  def init_client
+    fail 'please specify rubicon account id' unless @options['account_id']
+    @account_id = @options['account_id']
+  end
+
+  def before_quit_with_error
+  end
+
+  private
+
+  def scrap
+    tz = date.to_time.dst? ? '-07:00' : '-08:00'
+    date_str = date.strftime '%F'
+    date_start_str = "#{date_str}T00:00:00#{tz}"
+    date_end_str = "#{date_str}T23:59:59#{tz}"
+    byebug
+
+    response = RestClient::Request.execute method: :get, url: 'https://api.rubiconproject.com/analytics/v1/report/', user: @login, password: @secret, account: "publisher/#{@account_id}", start: date_start_str, end: date_end_str, dimensions: ['date'], metrics: ['revenue']
+
+    date_str = @date.strftime('%-m/%-d/%Y')
+    time_zone_id = 'Eastern Standard Time'
+
+    response = RestClient.post "https://ui.adsupply.com/PublicPortal/Publisher/#{@login}/Report/Export", SqlCommandId: '', ExportToExcel: 'False', IsOLAP: 'False', DateFilter: date_str, TimeZoneId: time_zone_id, Grouping: '1', 'DimPublisher.Value': "#{@login}~", 'DimPublisher.IsActive': 'True', 'DimSiteName.Value': '', 'DimSiteName.IsActive': 'True', 'DimCountry.Value': '', 'DimCountry.IsActive': 'True', 'DimMediaType.Value': '', 'DimMediaType.IsActive': 'True', ApiKey: @secret
+
+    data = JSON.parse response
+    header = data[0].keys
+    @data = [header]
+    @data += data.map do |datum|
+      header.map { |key| datum[key] }
+    end
+  end
+end

data/lib/adops_report_scrapper/undertone_client.rb

@@ -0,0 +1,85 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::UndertoneClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 3
+    false
+  end
+
+  private
+
+  def login
+    @client.visit 'https://insights.undertone.com/user/login'
+    @client.fill_in 'user_email', :with => @login
+    @client.fill_in 'user_password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[text()="Reporting"]'
+    rescue Exception => e
+      raise e, 'Undertone login error'
+    end
+  end
+
+  def scrap
+    request_zone_report
+    extract_data_from_zone_report
+    request_po_report
+    extract_data_from_po_report
+  end
+
+  def request_zone_report
+    date_str = date.strftime('%m/%d/%Y')
+    @client.fill_in 'start', :with => date_str
+    @client.fill_in 'end', :with => date_str
+    sleep 2
+    @client.select 'Zone Report - Publishers', :from => 'report_id'
+    sleep 2
+    @client.click_button 'html'
+    sleep 10
+  end
+
+  def extract_data_from_zone_report
+    @client.within_window @client.driver.browser.window_handles.last do
+      @data = []
+      rows = @client.find_all :xpath, '//table/*/tr[not(contains(@class, "text"))]'
+      rows = rows.to_a
+      rows.pop
+      @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+    end
+  end
+
+  def request_po_report
+    date_str = date.strftime('%m/%d/%Y')
+    @client.fill_in 'start', :with => date_str
+    @client.fill_in 'end', :with => date_str
+    sleep 2
+    @client.select 'Purchase Order Summary', :from => 'report_id'
+    sleep 2
+    @client.click_button 'html'
+    sleep 10
+  end
+
+  def extract_data_from_po_report
+    @client.within_window @client.driver.browser.window_handles.last do
+      rows = @client.find_all :xpath, '//table/*/tr[not(contains(@class, "text"))]'
+      rows = rows.to_a
+      rows.pop
+      rows.pop
+      keys = rows.shift
+      keys = keys.find_css('td,th').map { |td| td.visible_text }
+      valuess = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+      l_data = valuess.map { |values| Hash[keys.zip(values)] }
+      @data[0] << 'Revenue'
+      l_data.each do |l_datum|
+        @data.each_with_index do |datum, i|
+          next if i == 0
+          next unless datum[0].downcase.tr(' ','').include? "-#{l_datum['Ad Unit'].downcase.tr(' ','')}-"
+          @data[i] << datum[3].tr(',','').to_i * l_datum['Sales Price'].tr('USD','').to_f / 1000
+          break
+        end
+      end
+    end
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.36"
+  VERSION = "0.1.37"
 end

data/secret.sample.yml

@@ -100,3 +100,11 @@ rhythmone:
 littlethings:
   login: ------
   secret: ------
+rubicon:
+  login: ------
+  secret: ------
+  options:
+    account_id: ------
+undertone:
+  login: ------
+  secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.36
+  version: 0.1.37
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-08 00:00:00.000000000 Z
+date: 2016-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -241,10 +241,12 @@ files:
 - lib/adops_report_scrapper/openx_client.rb
 - lib/adops_report_scrapper/revcontent_client.rb
 - lib/adops_report_scrapper/rhythmone_client.rb
+- lib/adops_report_scrapper/rubicon_client.rb
 - lib/adops_report_scrapper/sonobi_client.rb
 - lib/adops_report_scrapper/springserve_client.rb
 - lib/adops_report_scrapper/tremor_client.rb
 - lib/adops_report_scrapper/triplelift_client.rb
+- lib/adops_report_scrapper/undertone_client.rb
 - lib/adops_report_scrapper/version.rb
 - lib/adops_report_scrapper/zedo_client.rb
 - secret.sample.yml