adops_report_scrapper 0.2.32 → 0.2.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 83f384aff65260cb64f8382b8a928dc383576b63
-  data.tar.gz: 75808e07c911e4d6249bd2c6feb619f653a756e7
+  metadata.gz: 2c691bbb4d4fe5920fbb4b2a718ca81ff36b710f
+  data.tar.gz: e3642185b69fd2bea0f62f61f5c29fc44b7dc62e
 SHA512:
-  metadata.gz: 377ebbfb40364b7692de2f400909d88bfe81bef3777936024c653800bc881ab8250d8b17d12a14de55204e039a012394f30ae58cd8ec3be0747781b4f369495c
-  data.tar.gz: af0e65a8ea7fe930acbaebcc338ade142b2882d5fd078832f1f3f48dc2b85ee42d3a0d52f83d8f514c4aedb08f074965abd5aef020d6f88a7bdf9ff03b3ff511
+  metadata.gz: a065b612f7dcee31d55758b12e2905359263f013051bb69ed9509bc1d40bdcaafa11c52c6c8f13af80077cc77dedae72f920d8ede7e9b37b98d48f96a855c84a
+  data.tar.gz: 01e743ce8bed604c08c717b02b354dd7bece0a22898421cf4227fab8f4429075368b1721effd0aa9fbc2bf673aae10786a20106181042c441dc409835df4f3d6

data/CHANGELOG

@@ -1,6 +1,9 @@
 # CHANGELOG
 
-## 0.2.31
+## 0.2.33
+add lkqd
+
+## 0.2.32
 reset localstorage before each run
 
 ## 0.2.30

data/Rakefile

@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
 require 'byebug'
 
 desc 'Collect all data'
-task :all => [:anyclip, :openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange] do # openx is the most unstable one, run it first
+task :all => [:anyclip, :openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange, :lkqd] do # openx is the most unstable one, run it first
   puts '========== You are all set'
 end
 
@@ -221,6 +221,11 @@ task :adsparc do
   save_as_csv :adsparc, :adsparc
 end
 
+desc 'Collect lkqd data'
+task :lkqd do
+  save_as_csv :lkqd, :lkqd
+end
+
 def date
   @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
 end

data/lib/adops_report_scrapper.rb

@@ -55,3 +55,4 @@ require 'adops_report_scrapper/spotxchange_client'
 require 'adops_report_scrapper/email_client'
 require 'adops_report_scrapper/anyclip_client'
 require 'adops_report_scrapper/adsparc_client'
+require 'adops_report_scrapper/lkqd_client'

data/lib/adops_report_scrapper/lkqd_client.rb

@@ -0,0 +1,71 @@
+require 'date'
+require_relative 'base_client'
+require 'securerandom'
+
+class AdopsReportScrapper::LkqdClient < AdopsReportScrapper::BaseClient
+  def date_supported?(date = nil)
+    _date = date || @date
+    return true if _date >= Date.today - 7
+    false
+  end
+
+  private
+
+  def login
+    @client.visit 'https://ui.lkqd.com/login'
+    @client.fill_in 'Username', :with => @login
+    @client.fill_in 'Password', :with => @secret
+    @client.click_button 'Sign In'
+    begin
+      @client.find :xpath, '//*[text()="Run Report"]'
+    rescue Exception => e
+      raise e, 'Lkqd login error'
+    end
+    cookies = @client.driver.cookies
+    @client = HTTPClient.new
+    @client.cookie_manager.cookies = cookies.values.map do |cookie|
+      cookie = cookie.instance_variable_get(:@attributes)
+      HTTP::Cookie.new cookie
+    end
+  end
+
+  def scrap
+    @date_str = @date.strftime('%Y-%m-%d')
+
+    header = {
+      'Content-Type': 'application/json'
+    }
+    body = {
+      'whatRequest': 'breakdown',
+      'uuid': SecureRandom.uuid,
+      'reportFormat': 'JSON',
+      'includeSummary': true,
+      'dateRangeType': 'CUSTOM',
+      'startDate': @date_str,
+      'endDate': @date_str,
+      'timeDimension': 'OVERALL',
+      'timezone': 'America/New_York',
+      'reportType': ['DOMAIN'],
+      'environmentIds': [1,2,3,4],
+      'filters': [],
+      'metrics': ["OPPORTUNITIES", "IMPRESSIONS", "REVENUE", "CLICKS", "COMPLETED_VIEWS"],
+      'sort': [{'field':'REVENUE','order':'desc'}],
+      'offset': 0,
+      'limit': 200
+    }
+    response = @client.post('https://ui-api.lkqd.com/reports', header: header, body: body.to_json)
+    data = JSON.parse(response.body)
+    data = data['data']['entries']
+
+    @data = []
+    return if data.count == 0
+
+    # flatten the dimensions and merge to top level of datum
+    data = data.map { |datum| datum.merge(datum['dimensions'].map { |d| [d['dimensionId'].downcase, d['name']] }.to_h) }
+    data = data.each { |datum| datum.delete('dimensions') }
+
+    header = data[0].keys
+    @data = [header]
+    @data += data.map { |datum| header.map { |key| datum[key].is_a?(Hash) ? datum[key]['value'] : datum[key] } }
+  end
+end

data/lib/adops_report_scrapper/version.rb

@@ -1,3 +1,3 @@
 module AdopsReportScrapper
-  VERSION = "0.2.32"
+  VERSION = "0.2.33"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: adops_report_scrapper
 version: !ruby/object:Gem::Version
-  version: 0.2.32
+  version: 0.2.33
 platform: ruby
 authors:
 - Stayman Hou
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-21 00:00:00.000000000 Z
+date: 2017-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -271,6 +271,7 @@ files:
 - lib/adops_report_scrapper/imonomy_client.rb
 - lib/adops_report_scrapper/littlethings_client.rb
 - lib/adops_report_scrapper/liveintent_client.rb
+- lib/adops_report_scrapper/lkqd_client.rb
 - lib/adops_report_scrapper/marfeel_client.rb
 - lib/adops_report_scrapper/nativo_client.rb
 - lib/adops_report_scrapper/netseer_client.rb