adops_report_scrapper 0.2.14 → 0.2.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG +3 -0
- data/Rakefile +6 -1
- data/lib/adops_report_scrapper.rb +1 -0
- data/lib/adops_report_scrapper/anyclip_client.rb +40 -0
- data/lib/adops_report_scrapper/version.rb +1 -1
- data/secret.sample.yml +5 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b956dd2e93da86703af0b79fe9ac00547c8d4080
|
4
|
+
data.tar.gz: 016d2f647d7c7fc3861b9613a2501cd996c2a4f5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 465fc6c18a54e8e3a06850aa1dfccfd33f8c68e033e316c6df206542f6bb14a1e13c3b39e7dfee999769ef488e8f01c98f5ea000b6f8ae554f40b8eebd2f54c6
|
7
|
+
data.tar.gz: f99077ba114bae2be8e78e5707e6c0c3c76c9330ec88ed7573492c4933587e929379cb58cba7b3d15ef89b815346777f650cb8e155e88f1a254c103b4d9ce3bb
|
data/CHANGELOG
CHANGED
data/Rakefile
CHANGED
@@ -7,7 +7,7 @@ require 'adops_report_scrapper'
|
|
7
7
|
require 'byebug'
|
8
8
|
|
9
9
|
desc 'Collect all data'
|
10
|
-
task :all => [:openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange] do # openx is the most unstable one, run it first
|
10
|
+
task :all => [:anyclip, :openx, :tremor, :brightroll, :yellowhammer, :adaptv, :fourninefive, :adx, :revcontent, :gcs, :browsi, :netseer, :sonobi, :nativo, :adsupply, :marfeel, :adsense, :criteo, :triplelift, :conversant, :liveintent, :adiply, :contentad, :facebookaudience, :adtechus, :adtomation, :rhythmone, :littlethings, :appnexus, :sovrn, :spotxchange] do # openx is the most unstable one, run it first
|
11
11
|
puts '========== You are all set'
|
12
12
|
end
|
13
13
|
|
@@ -201,6 +201,11 @@ task :mediabong do
|
|
201
201
|
save_as_csv :mediabong, :email
|
202
202
|
end
|
203
203
|
|
204
|
+
desc 'Collect anyclip data'
|
205
|
+
task :anyclip do
|
206
|
+
save_as_csv :anyclip, :anyclip
|
207
|
+
end
|
208
|
+
|
204
209
|
def date
|
205
210
|
@date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
|
206
211
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'date'
|
2
|
+
require_relative 'base_client'
|
3
|
+
|
4
|
+
class AdopsReportScrapper::AnyclipClient < AdopsReportScrapper::BaseClient
|
5
|
+
def date_supported?(date = nil)
|
6
|
+
_date = date || @date
|
7
|
+
return true if _date >= Date.today - 4
|
8
|
+
false
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def login
|
14
|
+
@client.driver.headers = { 'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' }
|
15
|
+
@client.visit 'https://partners.anyclip-media.com/#/signin'
|
16
|
+
sleep 10
|
17
|
+
return if @client.find_all(:xpath, '//*[text()="Network Activity Report"]').count > 0
|
18
|
+
@client.fill_in 'Username', :with => @login
|
19
|
+
@client.fill_in 'Password', :with => @secret
|
20
|
+
@client.find(:xpath, '//*[text()="LOG IN"]').click
|
21
|
+
begin
|
22
|
+
@client.find :xpath, '//*[text()="Network Activity Report"]'
|
23
|
+
rescue Exception => e
|
24
|
+
raise e, 'anyclip login error'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def scrap
|
29
|
+
date_str = @date.strftime '%Y-%m-%d'
|
30
|
+
@client.visit "https://partners.anyclip-media.com/#/report?dimension=trafficChannel&endDate=#{date_str}T23%3A59%3A59%2B00%3A00&sortAsc=false&sortBy=cost&startDate=#{date_str}T00%3A00%3A00%2B00%3A00&type=supply-partner-traffic-channel"
|
31
|
+
sleep 10
|
32
|
+
@data = []
|
33
|
+
@data << @client.find_all(:css, '.sr-collection--header .col').map { |cell| cell.text }
|
34
|
+
raw_data = @client.find_all(:css, '.sr-collection--row-item .col').map { |cell| cell.text }
|
35
|
+
(raw_data.count/6).times do |i|
|
36
|
+
si = raw_data.count/6+i*5
|
37
|
+
@data << [raw_data[i]] + raw_data[si, 5]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/secret.sample.yml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: adops_report_scrapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stayman Hou
|
@@ -253,6 +253,7 @@ files:
|
|
253
253
|
- lib/adops_report_scrapper/adtechus_client.rb
|
254
254
|
- lib/adops_report_scrapper/adtomation_client.rb
|
255
255
|
- lib/adops_report_scrapper/adx_client.rb
|
256
|
+
- lib/adops_report_scrapper/anyclip_client.rb
|
256
257
|
- lib/adops_report_scrapper/appnexus_client.rb
|
257
258
|
- lib/adops_report_scrapper/base_client.rb
|
258
259
|
- lib/adops_report_scrapper/brightcom_client.rb
|