adops_report_scrapper 0.1.15 → 0.1.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +5 -0
- data/lib/adops_report_scrapper/version.rb +1 -1
- data/lib/adops_report_scrapper/zedo_client.rb +45 -0
- data/lib/adops_report_scrapper.rb +1 -0
- data/secret.sample.yml +6 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 862be6e4eac422735934063bfbf6ae24941c399e
|
4
|
+
data.tar.gz: ff3d1ea7ae4317c21e0e95c7485d3c88399e66f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a89e0e93e8e2fb2d980d93ddb1494b13e0e4337a815cc34e43a118d57986ec95884615277d9719aa990917272046486b7165f88375f7731bc72f4b61ca8f439b
|
7
|
+
data.tar.gz: 318716e9c09ed18364b78186b5b1b2b855fa8b5bfedcae48692643f76b40212d79e28eac0fb738dc588d1f94e713a0b9c9781f006cddcd733d9dc20a9e174d0a
|
data/Rakefile
CHANGED
@@ -131,6 +131,11 @@ task :adtechus do
|
|
131
131
|
save_as_csv :adtechus, :adtechus
|
132
132
|
end
|
133
133
|
|
134
|
+
desc 'Collect divisiond data'
|
135
|
+
task :divisiond do
|
136
|
+
save_as_csv :divisiond, :zedo
|
137
|
+
end
|
138
|
+
|
134
139
|
def date
|
135
140
|
@date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
|
136
141
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'httpclient'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative 'base_client'
|
5
|
+
|
6
|
+
class AdopsReportScrapper::ZedoClient < AdopsReportScrapper::BaseClient
|
7
|
+
def date_supported?(date = nil)
|
8
|
+
_date = date || @date
|
9
|
+
return true if _date >= Date.today - 3
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def init_client
|
16
|
+
fail 'please specify zedo network_id' unless @options['network_id']
|
17
|
+
@network_id = @options['network_id']
|
18
|
+
@client = HTTPClient.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def login
|
22
|
+
response = @client.post('https://target.zedo.com/servlet/LoginServlet', uid: @login, pwd: @secret, rem: 0, from: 'login', fromSupport: 'no', fromEmail: 'false')
|
23
|
+
doc = Nokogiri::HTML(response.body)
|
24
|
+
profit_elem_href = doc.xpath('//a[text()="Profit"]/@href').first.value
|
25
|
+
@referer_url = "https://target.zedo.com#{profit_elem_href}"
|
26
|
+
@client.get(@referer_url)
|
27
|
+
end
|
28
|
+
|
29
|
+
def scrap
|
30
|
+
date_str = date.strftime('%m/%d/%Y')
|
31
|
+
|
32
|
+
body = {
|
33
|
+
step: 'submit', event_key: 'profit_rpt', mobilereport: 'off', scheduler: 'NO', schedule_event_key: '', time_period: '', reportTime: '', reporttypeName: 'Publisher Report', dateDifference: '', drilldown: 'false', domain: '', report: '0', tperiod: 'summary', timePeriod: 'summary', i18n_startDate: date_str, startDate: date_str, i18n_endDate: date_str, endDate: date_str, revenueTypeFilter: '-1', actionTypeFilter: 'post_total', publisher: '-1', channel: '-1', dimension: '-1', pageTime: Time.now.strftime('%s%3N'), nwtId: @network_id
|
34
|
+
}
|
35
|
+
header = { Referer: @referer_url }
|
36
|
+
|
37
|
+
response = @client.post('https://target.zedo.com/Main?reporttype=mbc&reportname=Quick_Profit_Report', body: body, header: header)
|
38
|
+
|
39
|
+
doc = Nokogiri::HTML(response.body)
|
40
|
+
header = doc.xpath('//table[@id="table-1"]/thead/tr/td').map { |td| td.text.strip }
|
41
|
+
@data = [header]
|
42
|
+
rows = doc.xpath('//table[@id="table-1"]/tbody/tr')
|
43
|
+
@data += rows.map { |tr| tr.css('td,th').map { |td| td.text.strip } }
|
44
|
+
end
|
45
|
+
end
|
data/secret.sample.yml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: adops_report_scrapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stayman Hou
|
@@ -242,6 +242,7 @@ files:
|
|
242
242
|
- lib/adops_report_scrapper/tremor_client.rb
|
243
243
|
- lib/adops_report_scrapper/triplelift_client.rb
|
244
244
|
- lib/adops_report_scrapper/version.rb
|
245
|
+
- lib/adops_report_scrapper/zedo_client.rb
|
245
246
|
- secret.sample.yml
|
246
247
|
- tmp/.keep
|
247
248
|
homepage: https://github.com/StaymanHou/adops_report_scrapper
|