adops_report_scrapper 0.2.32 → 0.2.33

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 83f384aff65260cb64f8382b8a928dc383576b63
4
- data.tar.gz: 75808e07c911e4d6249bd2c6feb619f653a756e7
3
+ metadata.gz: 2c691bbb4d4fe5920fbb4b2a718ca81ff36b710f
4
+ data.tar.gz: e3642185b69fd2bea0f62f61f5c29fc44b7dc62e
5
5
  SHA512:
6
- metadata.gz: 377ebbfb40364b7692de2f400909d88bfe81bef3777936024c653800bc881ab8250d8b17d12a14de55204e039a012394f30ae58cd8ec3be0747781b4f369495c
7
- data.tar.gz: af0e65a8ea7fe930acbaebcc338ade142b2882d5fd078832f1f3f48dc2b85ee42d3a0d52f83d8f514c4aedb08f074965abd5aef020d6f88a7bdf9ff03b3ff511
6
+ metadata.gz: a065b612f7dcee31d55758b12e2905359263f013051bb69ed9509bc1d40bdcaafa11c52c6c8f13af80077cc77dedae72f920d8ede7e9b37b98d48f96a855c84a
7
+ data.tar.gz: 01e743ce8bed604c08c717b02b354dd7bece0a22898421cf4227fab8f4429075368b1721effd0aa9fbc2bf673aae10786a20106181042c441dc409835df4f3d6
data/CHANGELOG CHANGED
@@ -1,6 +1,9 @@
1
1
  # CHANGELOG
2
2
 
3
- ## 0.2.31
3
+ ## 0.2.33
4
+ add lkqd
5
+
6
+ ## 0.2.32
4
7
  reset localstorage before each run
5
8
 
6
9
  ## 0.2.30
data/Rakefile CHANGED
@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
7
7
  require 'byebug'
8
8
 
9
9
  desc 'Collect all data'
10
- task :all => [:anyclip, :openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange] do # openx is the most unstable one, run it first
10
+ task :all => [:anyclip, :openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange, :lkqd] do # openx is the most unstable one, run it first
11
11
  puts '========== You are all set'
12
12
  end
13
13
 
@@ -221,6 +221,11 @@ task :adsparc do
221
221
  save_as_csv :adsparc, :adsparc
222
222
  end
223
223
 
224
+ desc 'Collect lkqd data'
225
+ task :lkqd do
226
+ save_as_csv :lkqd, :lkqd
227
+ end
228
+
224
229
  def date
225
230
  @date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
226
231
  end
@@ -55,3 +55,4 @@ require 'adops_report_scrapper/spotxchange_client'
55
55
  require 'adops_report_scrapper/email_client'
56
56
  require 'adops_report_scrapper/anyclip_client'
57
57
  require 'adops_report_scrapper/adsparc_client'
58
+ require 'adops_report_scrapper/lkqd_client'
@@ -0,0 +1,71 @@
1
+ require 'date'
2
+ require_relative 'base_client'
3
+ require 'securerandom'
4
+
5
+ class AdopsReportScrapper::LkqdClient < AdopsReportScrapper::BaseClient
6
+ def date_supported?(date = nil)
7
+ _date = date || @date
8
+ return true if _date >= Date.today - 7
9
+ false
10
+ end
11
+
12
+ private
13
+
14
+ def login
15
+ @client.visit 'https://ui.lkqd.com/login'
16
+ @client.fill_in 'Username', :with => @login
17
+ @client.fill_in 'Password', :with => @secret
18
+ @client.click_button 'Sign In'
19
+ begin
20
+ @client.find :xpath, '//*[text()="Run Report"]'
21
+ rescue Exception => e
22
+ raise e, 'Lkqd login error'
23
+ end
24
+ cookies = @client.driver.cookies
25
+ @client = HTTPClient.new
26
+ @client.cookie_manager.cookies = cookies.values.map do |cookie|
27
+ cookie = cookie.instance_variable_get(:@attributes)
28
+ HTTP::Cookie.new cookie
29
+ end
30
+ end
31
+
32
+ def scrap
33
+ @date_str = @date.strftime('%Y-%m-%d')
34
+
35
+ header = {
36
+ 'Content-Type': 'application/json'
37
+ }
38
+ body = {
39
+ 'whatRequest': 'breakdown',
40
+ 'uuid': SecureRandom.uuid,
41
+ 'reportFormat': 'JSON',
42
+ 'includeSummary': true,
43
+ 'dateRangeType': 'CUSTOM',
44
+ 'startDate': @date_str,
45
+ 'endDate': @date_str,
46
+ 'timeDimension': 'OVERALL',
47
+ 'timezone': 'America/New_York',
48
+ 'reportType': ['DOMAIN'],
49
+ 'environmentIds': [1,2,3,4],
50
+ 'filters': [],
51
+ 'metrics': ["OPPORTUNITIES", "IMPRESSIONS", "REVENUE", "CLICKS", "COMPLETED_VIEWS"],
52
+ 'sort': [{'field':'REVENUE','order':'desc'}],
53
+ 'offset': 0,
54
+ 'limit': 200
55
+ }
56
+ response = @client.post('https://ui-api.lkqd.com/reports', header: header, body: body.to_json)
57
+ data = JSON.parse(response.body)
58
+ data = data['data']['entries']
59
+
60
+ @data = []
61
+ return if data.count == 0
62
+
63
+ # flatten the dimensions and merge to top level of datum
64
+ data = data.map { |datum| datum.merge(datum['dimensions'].map { |d| [d['dimensionId'].downcase, d['name']] }.to_h) }
65
+ data = data.each { |datum| datum.delete('dimensions') }
66
+
67
+ header = data[0].keys
68
+ @data = [header]
69
+ @data += data.map { |datum| header.map { |key| datum[key].is_a?(Hash) ? datum[key]['value'] : datum[key] } }
70
+ end
71
+ end
@@ -1,3 +1,3 @@
1
1
  module AdopsReportScrapper
2
- VERSION = "0.2.32"
2
+ VERSION = "0.2.33"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: adops_report_scrapper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.32
4
+ version: 0.2.33
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stayman Hou
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-07-21 00:00:00.000000000 Z
11
+ date: 2017-07-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: httpclient
@@ -271,6 +271,7 @@ files:
271
271
  - lib/adops_report_scrapper/imonomy_client.rb
272
272
  - lib/adops_report_scrapper/littlethings_client.rb
273
273
  - lib/adops_report_scrapper/liveintent_client.rb
274
+ - lib/adops_report_scrapper/lkqd_client.rb
274
275
  - lib/adops_report_scrapper/marfeel_client.rb
275
276
  - lib/adops_report_scrapper/nativo_client.rb
276
277
  - lib/adops_report_scrapper/netseer_client.rb