adops_report_scrapper 0.1.65 → 0.1.66

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d27c37373f72fa370e6d798e0c3c26bab9f18cc
-  data.tar.gz: 0b7f49fc44cbaae730423e6b21aef0bd821aac4f
+  metadata.gz: 93a875311b6b77e868a4afb1498b6b4cb4e41d37
+  data.tar.gz: 05efa95ce31b91d04962cb27083983f43961de43
 SHA512:
-  metadata.gz: 83a8955ca204622903146bece36c436531cb029faf91053a33904afd318428a14b77e3299545b195eed8bb1cbf77a106b4a9a0383c487cd25af75a75e2c161c9
-  data.tar.gz: 5d1f0e254e9416803232a2af373c315528755ce0f01c4b759386facd0e32d86fc6199699d53c1c91fa732c3bfcdb75dfc25a4cdf53a623ac7c0c1b91716d7eb3
+  metadata.gz: bc4a06bcf1164aa5ad813820c574fe791474fee52f19f7626229018478807eb10d7397979fbf76c8a6a94b383d9f6a4a1647d64863854c11f9402ba14e7dbea9
+  data.tar.gz: 60e9ab3f00ff30bec0f3a652d18bfe2287d563b3b13c570d7288f56ef5e21f2e05268a35e68949e7e6f069b175033157291736ff74e46727391f48c9aa7fd216

data/CHANGELOG

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 0.1.66
+implement gumgum
+
 ## 0.1.65
 fix brightroll
 

data/Rakefile

@@ -186,6 +186,11 @@ task :sovrn do
   save_as_csv :sovrn, :sovrn
 end
 
+desc 'Collect gumgum data'
+task :gumgum do
+  save_as_csv :gumgum, :gumgum
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper.rb

@@ -48,3 +48,4 @@ require 'adops_report_scrapper/positivemobile_client'
 require 'adops_report_scrapper/brightcom_client'
 require 'adops_report_scrapper/appnexus_client'
 require 'adops_report_scrapper/sovrn_client'
+require 'adops_report_scrapper/gumgum_client'

data/lib/adops_report_scrapper/gumgum_client.rb

@@ -0,0 +1,81 @@
+require 'date'
+require_relative 'base_client'
+
+class AdopsReportScrapper::GumgumClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 7
+    false
+  end
+
+  private
+
+  def init_client
+    fail 'please specify gumgum product. e.g. in_image' unless @options['product']
+    @product = @options['product']
+    super
+  end
+
+  def login
+    @client.visit 'https://app.gumgum.com/login'
+    @client.fill_in 'Email', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Login'
+    begin
+      @client.find :xpath, '//*[text()="Reporting"]'
+    rescue Exception => e
+      raise e, 'Gumgum login error'
+    end
+    cookies = @client.driver.cookies
+    @client = HTTPClient.new
+    @client.cookie_manager.cookies = cookies.values.map do |cookie|
+      cookie = cookie.instance_variable_get(:@attributes)
+      HTTP::Cookie.new cookie
+    end
+  end
+
+  def scrap
+    @date_str = @date.strftime('%Y-%m-%d')
+    all_desktop_rows = get_parsed_report('ALL', '1')
+    all_mobile_rows = get_parsed_report('ALL', '2')
+    all_tablet_rows = get_parsed_report('ALL', '14')
+    us_desktop_rows = get_parsed_report('US', '1')
+    us_mobile_rows = get_parsed_report('US', '2')
+    us_tablet_rows = get_parsed_report('US', '14')
+    intl_desktop_rows = deduct_data_by_tracking_id(all_desktop_rows, us_desktop_rows)
+    intl_mobile_rows = deduct_data_by_tracking_id(all_mobile_rows, us_mobile_rows)
+    intl_tablet_rows = deduct_data_by_tracking_id(all_tablet_rows, us_tablet_rows)
+
+    @data = [['day', 'zoneName', 'trackingId', 'inventoryImpressions', 'adImpressions', 'earnings', 'country', 'browser']]
+    @data.concat(us_desktop_rows.map { |row| row.concat(['us', 'desktop']) })
+    @data.concat(us_mobile_rows.map { |row| row.concat(['us', 'mobile']) })
+    @data.concat(us_tablet_rows.map { |row| row.concat(['us', 'tablet']) })
+    @data.concat(intl_desktop_rows.map { |row| row.concat(['intl', 'desktop']) })
+    @data.concat(intl_mobile_rows.map { |row| row.concat(['intl', 'mobile']) })
+    @data.concat(intl_tablet_rows.map { |row| row.concat(['intl', 'tablet']) })
+  end
+
+  def get_parsed_report(country, brwoser)
+    response = @client.get("https://app.gumgum.com/p/reports/widget/table/metric/earnings/units/PRODUCT/target/#{@product}/start/#{@date_str}/end/#{@date_str}/zones/ALL/country/#{country}/browser/#{brwoser}/format/get.json")
+    return JSON.parse(response.body)['rows'].map do |row|
+      mapped_row = row.values_at(0, 1, 2, 3, 4, -1).map { |item| item['f'] }
+      mapped_row[3] = mapped_row[3].to_i
+      mapped_row[4] = mapped_row[4].to_i
+      mapped_row[5] = mapped_row[5].to_f
+      mapped_row
+    end
+  end
+
+  def deduct_data_by_tracking_id(rows_to_deduct, rows_deduct_by)
+    rows_to_return = rows_to_deduct.dup
+    rows_to_return.map do |row_to_return|
+      row_deduct_by = rows_deduct_by.select { |row_deduct_by| row_deduct_by[2] == row_to_return[2] }
+      return row_to_return if row_deduct_by.empty?
+      row_deduct_by = row_deduct_by[0]
+      row_to_return[3] = row_to_return[3] - row_deduct_by[3]
+      row_to_return[4] = row_to_return[4] - row_deduct_by[4]
+      row_to_return[5] = row_to_return[5] - row_deduct_by[5]
+      row_to_return
+    end
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.1.65"
+  VERSION = "0.1.66"
 end

data/secret.sample.yml

@@ -123,3 +123,8 @@ appnexus:
 sovrn:
   login: ------
   secret: ------
+gumgum:
+  login: ------
+  secret: ------
+  options:
+    product: ------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.1.65
+  version: 0.1.66
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-10 00:00:00.000000000 Z
+date: 2017-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -235,6 +235,7 @@ files:
 - lib/adops_report_scrapper/criteo_client.rb
 - lib/adops_report_scrapper/facebookaudience_client.rb
 - lib/adops_report_scrapper/gcs_client.rb
+- lib/adops_report_scrapper/gumgum_client.rb
 - lib/adops_report_scrapper/imonomy_client.rb
 - lib/adops_report_scrapper/littlethings_client.rb
 - lib/adops_report_scrapper/liveintent_client.rb