adops_report_scrapper 0.1.19 → 0.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 040bb4999a389ac0c4062bab0f3ea24ba1c4b7ee
-  data.tar.gz: abac49b6458516b4ae88288f2d25aa109b00274f
+  metadata.gz: 54ac8cdf2da487d3e2724718e72c3d27ca1139ce
+  data.tar.gz: d699316c998e1441290026fc8e689a67f5f16b64
 SHA512:
-  metadata.gz: 462a18889cee5235262fc44abb68fa2872eaf082b2b1f525d94994ade3c0f5041cb808ae97593c7d4011a8062e6e800c7bc2edc23a65b658544ea1dc6d1cf562
-  data.tar.gz: e4e03bf4f83411837ef3d81e948a4f34c1f3a02167685d21502d09aeed701f7631d131f7eb1c34945ee52d71d0ccbf1be9ce9d6308f66ccb70e1065a024fa466
+  metadata.gz: 07190bedabc4eced4195404044b7e40d6795e58f9948ef9df8156995fce278367dfdae3e20aa29f514630ddb24cfe0e8a07614ddcfc5d9cd608e36ca9769aae2
+  data.tar.gz: f539866ab7a944d64d6ddde8406f3da7825e4462dcdf0ef32bed1e88d645b6a4840a9d8990cffe0871170e93012c73d5a71b50a042042afd38ed1f102bbba53c

data/CHANGELOG

@@ -0,0 +1,7 @@
+# CHANGELOG
+
+## 0.1.20
+deprecate triplelift
+deprecate adforge
+implement rhythmone
+implement adtomation

data/Rakefile

@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
 require 'byebug'
 
 desc 'Collect all data'
-task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus] do # openx is the most unstable one, run it first
+task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone] do # openx is the most unstable one, run it first
   puts '========== You are all set'
 end
 
@@ -136,6 +136,16 @@ task :divisiond do
   save_as_csv :divisiond, :zedo
 end
 
+desc 'Collect adtomation data'
+task :adtomation do
+  save_as_csv :adtomation, :adtomation
+end
+
+desc 'Collect rhythmone data'
+task :rhythmone do
+  save_as_csv :rhythmone, :rhythmone
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper/adforge_client.rb

@@ -22,7 +22,10 @@ class AdopsReportScrapper::AdforgeClient < AdopsReportScrapper::BaseClient
   end
 
   def request_report
-    @client.find(:xpath, '//option[text()="Yesterday"]').select_option
+    @client.within_frame @client.find(:tag, :iframe) do
+      @client.find(:xpath, '//option[text()="Yesterday"]').select_option
+      @client.click_button 'Submit'
+    end
     sleep 5
   end
 

data/lib/adops_report_scrapper/adtomation_client.rb

@@ -0,0 +1,50 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::AdtomationClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'http://console.adtomation.com/'
+    @client.fill_in 'Username', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Sign In'
+
+    begin
+      @client.find :xpath, '//*[text()="Reporting"]'
+    rescue Exception => e
+      raise e, 'Adtomation login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.click_link 'Your Reports'
+    @client.find(:xpath, '//*[../../../../../td[text()="report created by adops report scrapper"] and contains(text(),"Run")]').click
+    sleep 5
+    wait_for_loading
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table/*/tr'
+    rows = rows.to_a
+    rows.pop
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+
+  def wait_for_loading
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:xpath, '//*[text()="Loading..."]')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 2
+  end
+end

data/lib/adops_report_scrapper/rhythmone_client.rb

@@ -0,0 +1,55 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::RhythmoneClient < AdopsReportScrapper::BaseClient
+  private
+
+  def login
+    @client.visit 'https://portal.rhythmone.com/login'
+    sleep 1
+    @client.fill_in 'email', :with => @login
+    @client.fill_in 'password', :with => @secret
+    @client.click_button 'Sign in'
+
+    begin
+      @client.find :xpath, '//*[contains(text(),"REPORTING")]'
+    rescue Exception => e
+      raise e, 'Rhythmone login error'
+    end
+  end
+
+  def scrap
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[contains(text(),"REPORTING")]').click
+    wait_for_loading
+    @client.find(:xpath, '//option[contains(text(),"Yesterday")]').select_option
+    sleep 1
+    @client.find(:xpath, '//select[@ng-model="filter.group1"]').find(:xpath, 'option[contains(text(),"Placement")]').select_option
+    wait_for_loading
+    @client.click_button 'Generate report'
+    sleep 1
+    wait_for_loading
+  end
+
+  def extract_data_from_report
+    rows = @client.find_all :xpath, '//table/*/tr'
+    rows = rows.to_a
+    @data = rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+  end
+
+  def wait_for_loading
+    30.times do |_i| # wait 5 min
+      begin
+        @client.find(:xpath, '//overlay-spinner/div')
+      rescue Exception => e
+        break
+      end
+      sleep 10
+    end
+    sleep 2
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.19"
+  VERSION = "0.1.20"
 end

data/lib/adops_report_scrapper.rb

@@ -14,7 +14,7 @@ module AdopsReportScrapper
 end
 
 require 'adops_report_scrapper/adaptv_client'
-require 'adops_report_scrapper/adforge_client'
+# require 'adops_report_scrapper/adforge_client'
 require 'adops_report_scrapper/adiply_client'
 require 'adops_report_scrapper/adsense_client'
 require 'adops_report_scrapper/adsupply_client'
@@ -35,6 +35,8 @@ require 'adops_report_scrapper/revcontent_client'
 require 'adops_report_scrapper/sonobi_client'
 require 'adops_report_scrapper/springserve_client'
 require 'adops_report_scrapper/tremor_client'
-require 'adops_report_scrapper/triplelift_client'
+# require 'adops_report_scrapper/triplelift_client'
 require 'adops_report_scrapper/adtechus_client'
 require 'adops_report_scrapper/zedo_client'
+require 'adops_report_scrapper/adtomation_client'
+require 'adops_report_scrapper/rhythmone_client'

data/secret.sample.yml

@@ -87,3 +87,9 @@ divisiond:
   secret: ------
   options:
     network_id: ------
+adtomation:
+  login: ------
+  secret: ------
+rhythmone:
+  login: ------
+  secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.20
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-06 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -207,6 +207,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -222,6 +223,7 @@ files:
 - lib/adops_report_scrapper/adsense_client.rb
 - lib/adops_report_scrapper/adsupply_client.rb
 - lib/adops_report_scrapper/adtechus_client.rb
+- lib/adops_report_scrapper/adtomation_client.rb
 - lib/adops_report_scrapper/adx_client.rb
 - lib/adops_report_scrapper/base_client.rb
 - lib/adops_report_scrapper/brightroll_client.rb
@@ -237,6 +239,7 @@ files:
 - lib/adops_report_scrapper/netseer_client.rb
 - lib/adops_report_scrapper/openx_client.rb
 - lib/adops_report_scrapper/revcontent_client.rb
+- lib/adops_report_scrapper/rhythmone_client.rb
 - lib/adops_report_scrapper/sonobi_client.rb
 - lib/adops_report_scrapper/springserve_client.rb
 - lib/adops_report_scrapper/tremor_client.rb