adops_report_scrapper 0.2.44 → 0.2.45

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 511782172f619bbe1c7b0e969aceb11b6866f0d3
-  data.tar.gz: 6a6abe3fb08919320422aad07248e301025c6b4f
+  metadata.gz: 6e398fac6bfeffceed28f29887aa913d02fd2942
+  data.tar.gz: 7498776ba363f9d5533ad8418641435274114718
 SHA512:
-  metadata.gz: 440c7c86d225157fcec55fd5e7580494fdac2d32764571345d996a6a1e31672c17d1afe00311981b9686610fbbb049681906e0e245b3f0630869ff9402dcd408
-  data.tar.gz: f231f1ebdf6dd5c7b4a346363333ab7193a7cb80c533ecbfba4b61132af487ba45347da990f10bb9fb16c0b69e284de0a39c05c0bedf80931e7d15dd4d4bba25
+  metadata.gz: 00493541a04fdbefd643278a45ab21c190ca71cb6eadafaf12346c8a1f5643b6179e72b4b573e9f4f2173955f1a214cb3d66f4c6105294b452fbc824f80e64fe
+  data.tar.gz: bf4723da5f70aeee4ea5ed12cc4728b6cf8fb34be7d72467242446723c98a5fb60770099cd78f526fa4da89fdbe3c3cf3936ba1d66970129a868d9dae8dc1bc5

data/CHANGELOG

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 0.2.45
+reimplement openx with their API
+
 ## 0.2.41
 springserve client bug fix for account_id option. revcontent client refactor using their rest api
 

data/Gemfile

@@ -18,3 +18,4 @@ gem 'cheddar'
 gem 'roo', '~> 2.4.0'
 gem 'selenium-webdriver', '~> 3.4.0'
 gem 'mail'
+gem 'oauth'

data/lib/adops_report_scrapper/openx_client.rb

@@ -1,165 +1,52 @@
 require 'date'
 require_relative 'base_client'
+require_relative '../helpers/ox3client'
+require 'csv'
 
 class AdopsReportScrapper::OpenxClient < AdopsReportScrapper::BaseClient
-  REPORT_NAME = 'Ad Server Report for adops_report_scrapper'
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 2
+    false
+  end
 
   private
 
-  def login
-    @client.driver.resize 1920, 700
-    fail 'please specify openx account prefix' unless @options['account_prefix']
-    @account_prefix = @options['account_prefix']
-    @client.visit "http://#{@account_prefix}.openx.net/"
-    @client.fill_in 'Email', :with => @login
-    @client.fill_in 'Password', :with => @secret
-    @client.click_button 'Submit'
-    begin
-      @client.find :xpath, '//*[text()="Reports"]'
-    rescue Exception => e
-      raise e, 'Openx login error'
-    end
-  end
-
-  def scrap
-    request_report
+  def init_client
+    fail 'please specify openx consumer_key' unless @options['consumer_key']
+    fail 'please specify openx consumer_secret' unless @options['consumer_secret']
+    fail 'please specify openx realm' unless @options['realm']
+    fail 'please specify openx site_url' unless @options['site_url']
+    @consumer_key = @options['consumer_key']
+    @consumer_secret = @options['consumer_secret']
+    @realm = @options['realm']
+    @site_url = @options['site_url']
   end
 
-  def request_report
-    sleep 5
-    @client.visit 'http://yahoo.com'
-    sleep 5
-    @client.visit "http://#{@account_prefix}.openx.net/#/reports?tab=my_reports"
-    sleep 5
-
-    begin
-      tries ||= 6
-      @client.find(:css, '#report_frame')
-    rescue Exception => e
-      retry unless (tries -= 1).zero?
-      fail 'cannot find report frame'
-    end
-
-    create_report_if_not_exist
-    
-    @client.within_frame @client.find(:css, '#report_frame') do
-      @client.find(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").click
-
-      begin
-        tries ||= 6
-        @client.find(:css, '.myFrame')
-      rescue Exception => e
-        retry unless (tries -= 1).zero?
-      end
-      
-      @client.within_frame @client.find(:css, '.myFrame') do
-        begin
-          tries ||= 18
-          @client.find(:xpath, '//option[text()="500"]')
-        rescue Exception => e
-          retry unless (tries -= 1).zero?
-        end
-        
-        @client.find(:xpath, '//option[text()="500"]').select_option
-        extract_data_from_report
-      end
-    end
-    sleep 5
+  def before_quit_with_error
   end
 
-  def create_report_if_not_exist
-    @client.within_frame @client.find(:css, '#report_frame') do
-      ready_elem = nil
-      begin
-        tries ||= 6
-        ready_elem = @client.find(:xpath, '//*[text()="Preconfigured Reports"]')
-      rescue Exception => e
-        retry unless (tries -= 1).zero?
-      end
-
-      fail 'openx report page not ready' unless ready_elem
+  def scrap
+    start_date_str = @date.strftime('%Y-%m-%d 00:00:00')
+    end_date_str = @date.strftime('%Y-%m-%d 23:59:59')
     
-      if @client.find_all(:xpath, "//a[text()=\"#{REPORT_NAME}\"]").count == 0
-
-        # create report if not exist
-        @client.find(:xpath, '//*[contains(text(),"Create Report")]').click
-        @client.find(:xpath, '//li/*[text()="Ad Server Report"]').click
+    ox3 = OX3APIClient.new(@login, @secret, @site_url, @consumer_key, @consumer_secret, @realm)
 
-        begin
-          tries ||= 6
-          @client.find(:css, '.myFrame')
-        rescue Exception => e
-          retry unless (tries -= 1).zero?
-        end
-        
-        @client.within_frame @client.find(:css, '.myFrame') do
+    response = ox3.get("/report/run?report=inv_rev&start_date=#{URI.escape(start_date_str)}&end_date=#{URI.escape(end_date_str)}&report_format=csv&do_break=AdUnit,Country&saleschannel=SALESCHANNEL.OPENXMARKET")
+    report_pickup_url = @site_url + JSON.parse(response)['url']
+    report_csv_data = nil;
+    open(report_pickup_url) { |f| report_csv_data = f.read }
 
-          @client.find(:xpath, '//*[text()="Last Seven Days"]').click
-          @client.find(:xpath, '//*[text()="Yesterday"]').click
-          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
-          sleep 2
+    @data = CSV.parse(report_csv_data)
 
-          @client.find(:xpath, '//*[text()="Ad Unit"]').click
-          @client.find(:xpath, '//img[../../td//*[text()="User"]]').click
-          @client.find(:xpath, '//*[text()="Country"]').click
-          @client.find(:xpath, '//*[text()="Device Category"]').click
-          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
-          sleep 2
-          
-          @client.find(:xpath, '//*[text()="Paid Impressions"]').click
-          @client.find(:xpath, '//*[text()="Impressions Delivered"]').click
-          @client.find(:xpath, '//*[text()="Ad Requests"]').click
-          @client.find(:xpath, '//img[../../td//*[text()="Revenue"]]').click
-          @client.find(:xpath, '//*[text()="Publisher Revenue"]').click
-          @client.find(:xpath, '//img[../../td//*[text()="Clicks"]]').click
-          @client.find(:xpath, '//div[text()="Clicks"]').click
-          @client.find(:xpath, '//*[@value="Next"]').trigger('click')
-          sleep 2
-
-          @client.find(:xpath, '//*[@value="Save"]').trigger('click')
-
-          begin
-            tries ||= 6
-            @client.find(:xpath, '//*[text()="Save Report"]')
-          rescue Exception => e
-            retry unless (tries -= 1).zero?
-          end
-          
-          @client.fill_in 'saveAsReportName', :with => REPORT_NAME
-          @client.find(:xpath, '//*[@value="Save Report"]').trigger('click')
-
-          begin
-            tries ||= 6
-            @client.find(:xpath, '//*[text()="Report Saved"]')
-          rescue Exception => e
-            retry unless (tries -= 1).zero?
-          end
-          
-          @client.find(:xpath, '//*[text()="Back to My Reports"]').trigger('click')
-        end
-        sleep 2
-      end
+    while @data.count > 0
+      row = @data.shift
+      break if row.first == 'Report Data:'
     end
-  end
 
-  def extract_data_from_report
-    @data = []
-    loop do
-      18.times do
-        sleep 10
-        break if @client.find_all(:xpath, '//table[@id="table_UniqueReportID"]/*/tr').count > 0
-      end
-      rows = @client.find_all :xpath, '//table[@id="table_UniqueReportID"]/*/tr'
-      rows = rows.to_a
-      header = rows.shift
-      if @data.count == 0
-        n_header = header.find_css('td,th').map { |td| td.visible_text }
-        @data << n_header
-      end
-      @data += rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }.reject { |row| row[0] == 'Total' }
-      pagee = @client.find(:xpath, '//*[contains(text(),"Showing rows")]').text.match(/-(\d+) of (\d+)\./).captures
-      break if pagee[0] == pagee[1]
-      @client.find_all(:css, '#paginationNext').first.trigger('click')
+    while @data.count > 0
+      break if @data.last.first == 'Real-time Buyer'
+      @data.pop
     end
   end
-end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.2.44"
+  VERSION = "0.2.45"
 end

data/lib/helpers/ox3client.rb

@@ -0,0 +1,184 @@
+# https://raw.githubusercontent.com/openx/OX3-Ruby-API-Client/master/ox3client.rb
+
+require 'oauth'
+require 'yaml'
+
+class OX3APIClient < OAuth::Consumer
+  
+  def initialize(email, password, site_url, consumer_key, consumer_secret, realm, 
+    version='v2', sso_domain='sso.openx.com', callback='oob', scheme='https', debug=false)
+    
+    @version, @callback, @site, @debug = version, callback, site_url, debug
+    @site = @site.end_with?('/') ? @site.chop : @site
+    
+    super(consumer_key, consumer_secret, {
+      :http_method => :post,
+      :scheme => :header,
+      :oauth_version => "1.0",
+      :signature_method => "HMAC-SHA1",
+      :site => @site,
+      :request_token_url => scheme + '://' + sso_domain + '/api/index/initiate',
+      :access_token_url => scheme + '://' + sso_domain + '/api/index/token',
+      :authorize_path => scheme + '://' + sso_domain + '/login/process',
+      :realm => realm
+    })
+    
+    # Step 1. Fetch temporary request token.
+    fetch_request_token
+    
+    # Step 2. Log in to SSO server and authorize token.
+    authorize_token(email, password)
+    
+    # Step 3. Swap temporary request token for permanent access token.
+    fetch_access_token
+    
+    # Step 4. Validate your access token.
+    validate_session
+  end
+
+################################################################################################################
+  def perform_request
+    http = self.create_http(@site)
+    http.open_timeout =  5 * 60
+    http.read_timeout = 15 * 60
+    params = Hash.new
+    params['Content-Type'] = 'application/json'
+    params['Cookie'] = 'openx3_access_token=' + @acccess_token.token + '; domain=' + get_domain + '; path=/'
+    prefix = "/ox/4.0"
+    response = http.request yield prefix, params
+    response.body
+  end
+
+  def get(path)
+    perform_request do |prefix, params|
+      self.create_http_request(
+        :get, 
+        prefix + path, 
+        params
+      )
+    end
+  end
+  
+  def post(path, body = {})
+    if body.is_a?(Hash)
+      body = JSON.dump(body)
+    end
+    perform_request do |prefix, params|
+      self.create_http_request(
+        :post, 
+        prefix + path,
+        body,
+        params
+      )
+    end
+  end
+  
+  def put(path, body = {})
+    if body.is_a?(Hash)
+      body = JSON.dump(body)
+    end
+    perform_request do |prefix, params|
+      self.create_http_request(
+        :put, 
+        prefix + path,
+        body,
+        params
+      )
+    end
+  end
+  
+  def delete(path)
+    perform_request do |prefix, params|
+      self.create_http_request(
+        :delete, 
+        prefix + path, 
+        params
+      )
+    end
+  end
+  
+  def logoff
+    get "/login/logout"
+  end
+################################################################################################################
+
+private
+  def fetch_request_token
+    @request_token = self.get_request_token(
+      {:oauth_callback => @callback}, 
+      {'Content-Type' => 'application/x-www-form-urlencoded'}
+    )
+    if @debug
+      puts YAML::dump(@request_token)
+    end
+  end
+  
+  def authorize_token(email, password)
+    authorize = self.request(:post,  self.authorize_path, nil, 
+      {:oauth_token => @request_token.token, :oauth_callback => @callback},
+      {:email => email, :password => password, :oauth_token => @request_token.token},
+      {'Content-Type' => 'application/x-www-form-urlencoded'}
+    )
+    if authorize.code.to_s == '200'
+      response = authorize
+    else
+      response = self.request(:post, authorize.header['Location'], nil, 
+        {:oauth_token => @request_token.token, :oauth_callback => @callback}, nil, 
+        {
+          'Content-Type' => 'application/x-www-form-urlencoded',
+          'Cookie' => authorize.get_fields('set-cookie')[0],
+        }
+      )
+    end
+    if @debug
+      puts YAML::dump(response)
+    end
+    @oauth_verifier = parse_tokens(response.body)[:oauth_verifier]
+  end
+  
+  def fetch_access_token
+    @acccess_token = @request_token.get_access_token(
+      {:oauth_verifier => @oauth_verifier, :oauth_token => @request_token.token, :oauth_callback => @callback},
+      {'Content-Type' => 'application/x-www-form-urlencoded'}
+    )
+    if @debug
+      puts YAML::dump(@acccess_token)
+    end
+  end
+  
+  def validate_session
+=begin
+    path = "/ox/4.0/session"
+    method = :get
+    response = self.request(method.to_sym, @site + path, nil, {}, nil,
+      {
+        'Content-Type' => 'application/json',
+        'Cookie' => 'openx3_access_token=' + @acccess_token.token + '; domain=' + get_domain + '; path=/'
+      }
+    )
+=end
+    response = perform_request do |prefix, params|
+      self.create_http_request(
+        :get, 
+        prefix + "/session",
+        nil,
+        params
+      )
+    end
+    if @debug
+      puts YAML::dump(response)
+    end
+  end
+  
+  def parse_tokens(keys)
+    keys.split("&").inject({}) do |hash, pair|
+      key, value = pair.split("=")
+      hash.merge({ key.to_sym => CGI.unescape(value) })
+    end
+  end
+  
+  def get_domain
+    URI.parse(@site).host
+  end
+  
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.2.44
+  version: 0.2.45
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-22 00:00:00.000000000 Z
+date: 2017-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -291,6 +291,7 @@ files:
 - lib/adops_report_scrapper/undertone_client.rb
 - lib/adops_report_scrapper/version.rb
 - lib/adops_report_scrapper/zedo_client.rb
+- lib/helpers/ox3client.rb
 - secret.sample.yml
 - tmp/.keep
 homepage: https://github.com/StaymanHou/adops_report_scrapper