adops_report_scrapper 0.1.45 → 0.1.46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 616a6781743ec9a8cc88ae394a52edd334fc9566
-  data.tar.gz: b4eb9f4fd40048b7909a78a174ba66ad384ec5c8
+  metadata.gz: 275a3babb56c7deaada93d0b5114ee6d03d12613
+  data.tar.gz: 9acf06c49ebc3b0bb50a3be33933bdea1a5a69d9
 SHA512:
-  metadata.gz: 3e69e6824da561cd1df24c00b9056be7991d710b3e85f195ea923a212e5a91ee4ef0f6f41362100dd1cd5089859c24d58e9e56ce85a87ce06c820cecfe002677
-  data.tar.gz: ea9e452aa8130c06825a08eac1b07749a701c437d58bab259220b925ef95ff3fc94d607085dab90efee03ad9710dc7e04755782aeef0f2dd29b83717cd903212
+  metadata.gz: f3649d81f5520ef417f05ecccead341985cd048244757ef690b16fdb2d55a50e8e1c9babdcbe50249b5cb266f8103f6249be174e66084cd2f32a6250cd987231
+  data.tar.gz: e4f2fc8a5bc7d919cb8481d664834457a1ef563b6da560ebb777c463b951512ec06dd3ff444660c7a6a212de1b33fe419fb5864a1be012a3f121c5d0d33302a5

data/CHANGELOG

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 0.1.46
+implement imonomy
+
 ## 0.1.44
 fixed tremor
 

data/README.md

@@ -1,6 +1,6 @@
 # Adops_Report_Scrapper
 
-Adops Report Scrapper is a collection of web scrappers that can automatically extract the data from your ad server, ad networks, ad exchanges, and other ad partners. It is an open source alternative to [STAQ](http://www.staq.com/). Say good-bye to spreadsheet. At the moment, this gem only focus on getting the data in. It is up to you how you would aggregate the data into your pageview & dfp/ad server data set.
+Adops Report Scrapper (I meant scraper :P) is a collection of web scrapers that can automatically extract the data from your ad server, ad networks, ad exchanges, and other ad partners. It is an open source alternative to [STAQ](http://www.staq.com/). Say good-bye to spreadsheet. At the moment, this gem only focus on getting the data in. It is up to you how you would aggregate the data into your pageview & dfp/ad server data set.
 
 ## Installation
 
@@ -30,11 +30,23 @@ AdopsReportScrapper.get_scrapper(:brightroll, 'login', 'secret', {}).get_data
 # example: [["Watch", "Status", "Name", "ID", "CPM Floor", "Revenue", "eCPM", "Requests", "Responses", "Response Rate", "Impressions", "Impression Rate", "Fill Rate", "Viewable Rate", "Clicks", "CTR", "Mid Rate", "Completion Rate", "Companion Impressions", "Companion Clicks", "Companion CTR", "Created On", "Last Modified"], ["", "", "My Post RON CDN", "3863603", "$6.00", "$9.99", "$6.68", "14,951", "2,556", "17.10%", "1,496", "58.53%", "10.01%", "60.87%", "7", "0.47%", "51.80%", "45.05%", "0.00%", "0", "0.00%", "03/23/2015", "06/25/2016"], ["", "", "My Post 400x300", "3855941", "$6.00", "$0.04", "$6.07", "78", "37", "47.44%", "6", "16.22%", "7.69%", "0.00%", "0", "0.00%", "50.00%", "66.67%", "0.00%", "0", "0.00%", "12/20/2013", "06/24/2016"], ["", "", "My Post", "3860218", "$7.00", "$0.03", "$8.07", "26", "12", "46.15%", "4", "33.33%", "15.38%", "0.00%", "0", "0.00%", "125.00%", "125.00%", "0.00%", "0", "0.00%", "10/10/2014", "06/09/2016"], ["", "", "My Post 640x480", "3855297", "$7.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "11/25/2013", "06/28/2016"], ["", "", "My Post 300x250", "3855943", "$2.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "12/20/2013", "06/09/2016"], ["", "", "Mobile - iPhone web…e My Post NCT", "3856072", "$4.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "01/07/2014", "05/23/2016"], ["", "", "Mobile - Android we…- The My Post", "3856089", "$4.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "01/08/2014", "05/23/2016"], ["", "", "Mobile - iPad web -…e My Post NCT", "3856127", "$4.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "01/10/2014", "05/23/2016"], ["", "", "My Today 6836382", "3860216", "$7.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "10/10/2014", "06/29/2016"], ["", "", "My Herald", "3860220", "$6.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "10/10/2014", "06/09/2016"], ["", "", "My News", "3860222", "$7.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "10/10/2014", "06/29/2016"], ["", "", "My Blog", "3860224", "$5.00", "$0.00", "$0.00", "15", "7", "46.67%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "10/10/2014", "06/09/2016"], ["", "", "My Times 400x300", "3863146", "$7.00", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "03/02/2015", "06/29/2016"], ["", "", "My Post RON UK", "3863604", "$5.46", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "03/23/2015", "06/09/2016"], ["", "", "My Post RON AU", "3863906", "$4.43", "$0.00", "$0.00", "0", "0", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0", "0.00%", "0.00%", "0.00%", "0.00%", "0", "0.00%", "04/01/2015", "06/09/2016"]]
 ```
 
-## Supported Scrappers
+## Standalone Setup
 
-Check `lib/adops_report_scrapper` directory for implemented scrappers.
+You may also choose to run this in the standalone mode.
 
-Check `secret.sample.yml` for expected args to be passed to each scrapper.
+Clone the code `git clone https://github.com/StaymanHou/adops_report_scrapper.git`
+
+Copy the configuration file `cp secret.sample.yml secret.yml`. And modify the credentials in the secret.yml accordingly.
+
+Install the dependencies `bundle install`
+
+Run the scraper `bundle exec rake <scraper_name> # e.g. bundle exec rake openx`. It will scrape yesterday's data, and save it as a CSV file in the `./tmp/` directory.
+
+## Supported Scrapers
+
+Check `lib/adops_report_scrapper` directory for implemented scrapers.
+
+Check `secret.sample.yml` for expected args to be passed to each scraper.
 
 ## Development
 

data/Rakefile

@@ -161,6 +161,11 @@ task :undertone do
   save_as_csv :undertone, :undertone
 end
 
+desc 'Collect imonomy data'
+task :imonomy do
+  save_as_csv :imonomy, :imonomy
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper.rb

@@ -43,3 +43,4 @@ require 'adops_report_scrapper/rhythmone_client'
 require 'adops_report_scrapper/littlethings_client'
 require 'adops_report_scrapper/rubicon_client'
 require 'adops_report_scrapper/undertone_client'
+require 'adops_report_scrapper/imonomy_client'

data/lib/adops_report_scrapper/imonomy_client.rb

@@ -0,0 +1,63 @@
+require 'date'
+require_relative 'base_client'
+
+# gcs sometimes doesn't update data in 24 hours
+
+class AdopsReportScrapper::ImonomyClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 7
+    false
+  end
+
+  private
+
+  def login
+    @client.visit 'http://dashboard.imonomy.com/'
+    @client.fill_in 'Username', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[text()="My Statistics"]'
+    rescue Exception => e
+      raise e, 'Imonomy login error'
+    end
+  end
+
+  def scrap
+    @data = []
+    return if @date == Date.today - 1 # imonomy never update revenues for in 1 day
+    request_report
+    extract_data_from_report
+  end
+
+  def request_report
+    @client.find(:xpath, '//*[text()="My Statistics"]').click
+
+    @client.find(:xpath, '//option[text()="Last 7 Days"]').select_option
+    @client.check 'Website'
+    @client.check 'Country'
+    @client.check 'Device'
+
+    @client.click_link 'Submit'
+    sleep 10
+
+    @client.find(:xpath, '//option[text()="100"]').select_option
+  end
+
+  def extract_data_from_report
+    until @client.find(:xpath, '//*[text()=">"]')[:disabled] == "disabled"
+      rows = @client.find_all :xpath, %Q{//table/tbody/tr[./td[text()="#{@date.strftime('%d.%m.%Y')}"]]}
+      rows = rows.to_a
+      if @data.count == 0
+        header = @client.find :xpath, '//table/thead/tr'
+        n_header = header.find_css('td,th').map { |td| td.visible_text }
+        @data << n_header
+      end
+      @data.concat rows.map { |tr| tr.find_css('td,th').map { |td| td.visible_text } }
+
+      @client.find(:xpath, '//*[text()=">"]').click
+      sleep 1
+    end
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.45"
+  VERSION = "0.1.46"
 end

data/secret.sample.yml

@@ -108,3 +108,6 @@ rubicon:
 undertone:
   login: ------
   secret: ------
+imonomy:
+  login: ------
+  secret: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.45
+  version: 0.1.46
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-13 00:00:00.000000000 Z
+date: 2017-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -233,6 +233,7 @@ files:
 - lib/adops_report_scrapper/criteo_client.rb
 - lib/adops_report_scrapper/facebookaudience_client.rb
 - lib/adops_report_scrapper/gcs_client.rb
+- lib/adops_report_scrapper/imonomy_client.rb
 - lib/adops_report_scrapper/littlethings_client.rb
 - lib/adops_report_scrapper/liveintent_client.rb
 - lib/adops_report_scrapper/marfeel_client.rb