adops_report_scrapper 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +5 -0
- data/lib/adops_report_scrapper/version.rb +1 -1
- data/lib/adops_report_scrapper/zedo_client.rb +45 -0
- data/lib/adops_report_scrapper.rb +1 -0
- data/secret.sample.yml +6 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 862be6e4eac422735934063bfbf6ae24941c399e
|
4
|
+
data.tar.gz: ff3d1ea7ae4317c21e0e95c7485d3c88399e66f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a89e0e93e8e2fb2d980d93ddb1494b13e0e4337a815cc34e43a118d57986ec95884615277d9719aa990917272046486b7165f88375f7731bc72f4b61ca8f439b
|
7
|
+
data.tar.gz: 318716e9c09ed18364b78186b5b1b2b855fa8b5bfedcae48692643f76b40212d79e28eac0fb738dc588d1f94e713a0b9c9781f006cddcd733d9dc20a9e174d0a
|
data/Rakefile
CHANGED
@@ -131,6 +131,11 @@ task :adtechus do
|
|
131
131
|
save_as_csv :adtechus, :adtechus
|
132
132
|
end
|
133
133
|
|
134
|
+
desc 'Collect divisiond data'
|
135
|
+
task :divisiond do
|
136
|
+
save_as_csv :divisiond, :zedo
|
137
|
+
end
|
138
|
+
|
134
139
|
def date
|
135
140
|
@date ||= ENV['date'].nil? ? Date.today - 1 : Date.today - ENV['date'].to_i
|
136
141
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'httpclient'
|
3
|
+
require 'nokogiri'
|
4
|
+
require_relative 'base_client'
|
5
|
+
|
6
|
+
class AdopsReportScrapper::ZedoClient < AdopsReportScrapper::BaseClient
|
7
|
+
def date_supported?(date = nil)
|
8
|
+
_date = date || @date
|
9
|
+
return true if _date >= Date.today - 3
|
10
|
+
false
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def init_client
|
16
|
+
fail 'please specify zedo network_id' unless @options['network_id']
|
17
|
+
@network_id = @options['network_id']
|
18
|
+
@client = HTTPClient.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def login
|
22
|
+
response = @client.post('https://target.zedo.com/servlet/LoginServlet', uid: @login, pwd: @secret, rem: 0, from: 'login', fromSupport: 'no', fromEmail: 'false')
|
23
|
+
doc = Nokogiri::HTML(response.body)
|
24
|
+
profit_elem_href = doc.xpath('//a[text()="Profit"]/@href').first.value
|
25
|
+
@referer_url = "https://target.zedo.com#{profit_elem_href}"
|
26
|
+
@client.get(@referer_url)
|
27
|
+
end
|
28
|
+
|
29
|
+
def scrap
|
30
|
+
date_str = date.strftime('%m/%d/%Y')
|
31
|
+
|
32
|
+
body = {
|
33
|
+
step: 'submit', event_key: 'profit_rpt', mobilereport: 'off', scheduler: 'NO', schedule_event_key: '', time_period: '', reportTime: '', reporttypeName: 'Publisher Report', dateDifference: '', drilldown: 'false', domain: '', report: '0', tperiod: 'summary', timePeriod: 'summary', i18n_startDate: date_str, startDate: date_str, i18n_endDate: date_str, endDate: date_str, revenueTypeFilter: '-1', actionTypeFilter: 'post_total', publisher: '-1', channel: '-1', dimension: '-1', pageTime: Time.now.strftime('%s%3N'), nwtId: @network_id
|
34
|
+
}
|
35
|
+
header = { Referer: @referer_url }
|
36
|
+
|
37
|
+
response = @client.post('https://target.zedo.com/Main?reporttype=mbc&reportname=Quick_Profit_Report', body: body, header: header)
|
38
|
+
|
39
|
+
doc = Nokogiri::HTML(response.body)
|
40
|
+
header = doc.xpath('//table[@id="table-1"]/thead/tr/td').map { |td| td.text.strip }
|
41
|
+
@data = [header]
|
42
|
+
rows = doc.xpath('//table[@id="table-1"]/tbody/tr')
|
43
|
+
@data += rows.map { |tr| tr.css('td,th').map { |td| td.text.strip } }
|
44
|
+
end
|
45
|
+
end
|
data/secret.sample.yml
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: adops_report_scrapper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stayman Hou
|
@@ -242,6 +242,7 @@ files:
|
|
242
242
|
- lib/adops_report_scrapper/tremor_client.rb
|
243
243
|
- lib/adops_report_scrapper/triplelift_client.rb
|
244
244
|
- lib/adops_report_scrapper/version.rb
|
245
|
+
- lib/adops_report_scrapper/zedo_client.rb
|
245
246
|
- secret.sample.yml
|
246
247
|
- tmp/.keep
|
247
248
|
homepage: https://github.com/StaymanHou/adops_report_scrapper
|