adops_report_scrapper 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90ddebdbcf3cfd0c810e901e848db5745beb16bf
-  data.tar.gz: 9e9974b8b6c28c3cebce1a9f7e104ff6bf714e1d
+  metadata.gz: edbe42b9645b997220cd4ac89c2d0473445eb11f
+  data.tar.gz: d3c6d4a8fd8668879b64e6bd0a46e2cc60dd077f
 SHA512:
-  metadata.gz: ca85fd4ca169052d8c34f865ea311c6c1c804cb6820c56637b40eff542c6086c73296fe5147ed0b7367ddbcd74777e8c587990b7f76f932199d7586a8515a4e8
-  data.tar.gz: 1874fbb0593d472c3f24339966008471cf84bed45e78b6b53c13e666ff56fa59df7749fd2210c4bb78aefae5eda5488f68089acddaf59708013ed98e0800b162
+  metadata.gz: ca089b75280f02a1d9af9a50d7d53b3975e1982541220842acf8955eda005ee13eba45956fd6b11bdcb65342850c3675383d2a141a6c565207818c3608419a70
+  data.tar.gz: e869dfcac65dbcfca94d924105ab4f26364575845a4ad166fb660bf4c73ff0dabeb3510292440e8f8455bc7f04d1ca38644e32648bf9c047e056676e6d57f6fc

data/Rakefile

@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
 require 'byebug'
 
 desc 'Collect all data'
-task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience] do # openx is the most unstable one, run it first
+task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus] do # openx is the most unstable one, run it first
   puts '========== You are all set'
 end
 
@@ -126,6 +126,11 @@ task :facebookaudience do
   save_as_csv :facebookaudience, :facebookaudience
 end
 
+desc 'Collect adtechus data'
+task :adtechus do
+  save_as_csv :adtechus, :adtechus
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper/adtechus_client.rb

@@ -0,0 +1,54 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::AdtechusClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://marketplace.adtechus.com'
+    @client.fill_in 'Username', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Sign in'
+    begin
+      @client.find :css, '#mainwindow'
+    rescue Exception => e
+      raise e, 'Adtechus login error'
+    end
+  end
+
+  def scrap
+    request_report
+  end
+
+  def request_report
+    @client.visit(@client.find(:css, '#mainwindow')[:src])
+    wait_for_loading
+    report_id = @client.find(:xpath, '//tr[./td/div/span[text()="Placement fill rate report"]]')[:id]
+    report_id = report_id.tr 'row_', ''
+    @client.visit "https://marketplace.adtechus.com/h2/reporting/showReport.do?action=showreportpage._._.#{report_id}"
+    @client.within_frame @client.find(:css, '#reportwindow') do
+      @client.within_frame @client.find(:xpath, '//iframe[@name="uid_2"]') do
+        extract_data_from_report
+      end
+    end
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table[@class="table"]/tbody/tr'
+    rows = rows.to_a
+    rows.pop
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+
+  def wait_for_loading
+    18.times do |_i| # wait 3 min
+      begin
+        @client.find(:xpath, '//*[contains(text(),"loading")]')
+      rescue Exception => e
+        break
+      end
+      sleep 3
+    end
+    sleep 1
+  end
+end

data/lib/adops_report_scrapper/gcs_client.rb

@@ -18,6 +18,11 @@ class AdopsReportScrapper::GcsClient < AdopsReportScrapper::BaseClient
     @client.click_button 'Next'
     @client.fill_in 'Passwd', :with => @secret
     @client.click_button 'Sign in'
+    # for veirfication
+    # cc = @client.find :xpath, '//*[contains(text(),"the recovery phone")]'
+    # cc.click
+    # @client.fill_in 'Enter phone number', :with => @options[:recovery_phone]
+    # @client.click_button 'Done'
     begin
       @client.find :xpath, '//*[text()="Sites"]'
     rescue Exception => e

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/lib/adops_report_scrapper.rb

@@ -36,3 +36,4 @@ require 'adops_report_scrapper/sonobi_client'
 require 'adops_report_scrapper/springserve_client'
 require 'adops_report_scrapper/tremor_client'
 require 'adops_report_scrapper/triplelift_client'
+require 'adops_report_scrapper/adtechus_client'

data/secret.sample.yml

@@ -74,4 +74,7 @@ facebookaudience:
   login: ------
   secret: ------
   options:
-    app_id: ------
+    app_id: ------
+adtechus:
+  login: ------
+  secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-23 00:00:00.000000000 Z
+date: 2016-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -221,6 +221,7 @@ files:
 - lib/adops_report_scrapper/adiply_client.rb
 - lib/adops_report_scrapper/adsense_client.rb
 - lib/adops_report_scrapper/adsupply_client.rb
+- lib/adops_report_scrapper/adtechus_client.rb
 - lib/adops_report_scrapper/adx_client.rb
 - lib/adops_report_scrapper/base_client.rb
 - lib/adops_report_scrapper/brightroll_client.rb