embulk-input-lkqd 0.0.0 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +20 -9
- data/embulk-input-lkqd.gemspec +5 -3
- data/lib/embulk/input/lkqd.rb +131 -24
- metadata +29 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b93579065f9e23f8657ba1347106a632be11d206
|
4
|
+
data.tar.gz: 68a43225355600b5f32a5cc747c901131615c70b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76e8dd612eb8ef05230b14dfff0a6c6af4d512cc9555a7429613fdbc01d217004efe06266eb1afe44167a6e63673080fcd3f9e82dd17bb9b59d2a507a5d49e25
|
7
|
+
data.tar.gz: 792d5f855f244a9c36752991c912f66862b90b41d69f9bbd655f21558c489f95b6cd4f2c55ce04ae81725dec96a11ea7cfdd141771f90aa1de1fe0f57fa4a5ef
|
data/README.md
CHANGED
@@ -1,27 +1,38 @@
|
|
1
1
|
# Lkqd input plugin for Embulk
|
2
2
|
|
3
|
-
|
3
|
+
Loads reporting data from LKQD API.
|
4
4
|
|
5
5
|
## Overview
|
6
6
|
|
7
7
|
* **Plugin type**: input
|
8
|
-
* **Resume supported**:
|
9
|
-
* **Cleanup supported**:
|
10
|
-
* **Guess supported**:
|
8
|
+
* **Resume supported**: no
|
9
|
+
* **Cleanup supported**: no
|
10
|
+
* **Guess supported**: yes
|
11
11
|
|
12
12
|
## Configuration
|
13
13
|
|
14
|
-
- **
|
15
|
-
- **
|
16
|
-
- **
|
14
|
+
- **secret_key_id**: API Secret Key ID (string, required)
|
15
|
+
- **secret_key**: API Secret Key (string, required)
|
16
|
+
- **endpoint**: API endpoint (string, default: `'https://api.lkqd.com/reports'`)
|
17
|
+
- **report_parameters**: Report parameters documented in https://wiki.lkqd.com/display/API/LKQD+API (hash, default: `{}`)
|
17
18
|
|
18
19
|
## Example
|
19
20
|
|
20
21
|
```yaml
|
21
22
|
in:
|
22
23
|
type: lkqd
|
23
|
-
|
24
|
-
|
24
|
+
secret_key_id:
|
25
|
+
secret_key:
|
26
|
+
report_parameters:
|
27
|
+
timeDimension: "HOURLY"
|
28
|
+
reportType: [""]
|
29
|
+
reportFormat: "CSV"
|
30
|
+
startDate: "2017-08-01"
|
31
|
+
endDate: "2017-08-02"
|
32
|
+
startHour: 0
|
33
|
+
endHour: 23
|
34
|
+
timezone: "UTC"
|
35
|
+
metrics: [""]
|
25
36
|
```
|
26
37
|
|
27
38
|
|
data/embulk-input-lkqd.gemspec
CHANGED
@@ -1,19 +1,21 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-input-lkqd"
|
3
|
-
spec.version = "0.
|
3
|
+
spec.version = "0.1.0"
|
4
4
|
spec.authors = ["Ming Liu"]
|
5
5
|
spec.summary = "LKQD input plugin for Embulk"
|
6
6
|
spec.description = "Loads reporting data from LKQD API."
|
7
7
|
spec.email = ["liuming@lmws.net"]
|
8
8
|
spec.licenses = ["MIT"]
|
9
|
-
|
9
|
+
spec.homepage = "https://github.com/liuming/embulk-input-lkqd"
|
10
10
|
|
11
11
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
12
12
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
13
13
|
spec.require_paths = ["lib"]
|
14
14
|
|
15
|
+
spec.add_dependency 'http', ['~> 2.2.2']
|
16
|
+
|
15
17
|
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
16
|
-
spec.add_development_dependency 'embulk', ['
|
18
|
+
spec.add_development_dependency 'embulk', ['= 0.8.28']
|
17
19
|
spec.add_development_dependency 'bundler', ['>= 1.10.6']
|
18
20
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
19
21
|
end
|
data/lib/embulk/input/lkqd.rb
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'json'
|
3
|
+
require 'csv'
|
4
|
+
require 'tempfile'
|
5
|
+
require 'http'
|
6
|
+
|
1
7
|
module Embulk
|
2
8
|
module Input
|
3
9
|
|
@@ -5,18 +11,25 @@ module Embulk
|
|
5
11
|
Plugin.register_input("lkqd", self)
|
6
12
|
|
7
13
|
def self.transaction(config, &control)
|
8
|
-
# configuration code:
|
9
14
|
task = {
|
10
|
-
"
|
11
|
-
"
|
12
|
-
"
|
15
|
+
"secret_key_id" => config.param("secret_key_id", :string),
|
16
|
+
"secret_key" => config.param("secret_key", :string),
|
17
|
+
"endpoint" => config.param("endpoint", :string, default: 'https://api.lkqd.com/reports'),
|
18
|
+
"report_parameters" => config.param("report_parameters", :hash, default: {}),
|
13
19
|
}
|
20
|
+
task['authorization'] = Base64.urlsafe_encode64("#{task['secret_key_id']}:#{task['secret_key']}")
|
21
|
+
|
22
|
+
response = request_lkqd({authorization: task['authorization'], endpoint: task['endpoint'], report_parameters: task['report_parameters']})
|
23
|
+
tempfile = Tempfile.new('lkqd_')
|
24
|
+
tempfile.write response.body
|
25
|
+
task['tempfile_path'] = tempfile.path
|
26
|
+
tempfile.close
|
14
27
|
|
15
|
-
columns = [
|
16
|
-
|
17
|
-
|
18
|
-
Column.new(
|
19
|
-
|
28
|
+
columns = []
|
29
|
+
::CSV.foreach(tempfile.path).first.each_with_index do |column_name, index|
|
30
|
+
column_type = guess_column_type(task['report_parameters']['metrics'], column_name)
|
31
|
+
columns << Column.new({index: index}.merge(column_type))
|
32
|
+
end
|
20
33
|
|
21
34
|
resume(task, columns, 1, &control)
|
22
35
|
end
|
@@ -28,32 +41,126 @@ module Embulk
|
|
28
41
|
return next_config_diff
|
29
42
|
end
|
30
43
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
44
|
+
def self.guess(config)
|
45
|
+
return {}
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.request_lkqd(config)
|
49
|
+
::HTTP.auth("Basic #{config[:authorization]}").post(config[:endpoint], json: config[:report_parameters])
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.guess_column_type(metrics, column_name)
|
53
|
+
column_name.gsub!(/^\W/, '')
|
54
|
+
column_option = DEFAULT_COLUMNS[column_name]
|
55
|
+
if column_option
|
56
|
+
return {type: column_option['out_type'].to_sym, name: column_name, format: column_option['format']}
|
57
|
+
else
|
58
|
+
return {type: :string, name: column_name}
|
59
|
+
end
|
60
|
+
end
|
40
61
|
|
41
62
|
def init
|
42
|
-
|
43
|
-
@option1 = task["option1"]
|
44
|
-
@option2 = task["option2"]
|
45
|
-
@option3 = task["option3"]
|
63
|
+
@task = task
|
46
64
|
end
|
47
65
|
|
48
66
|
def run
|
49
|
-
|
50
|
-
|
67
|
+
CSV.foreach(@task['tempfile_path'], {headers: true}).each do |row|
|
68
|
+
page_builder.add(row)
|
69
|
+
end
|
51
70
|
page_builder.finish
|
52
71
|
|
53
72
|
task_report = {}
|
54
73
|
return task_report
|
55
74
|
end
|
75
|
+
|
76
|
+
DEFAULT_COLUMNS = {
|
77
|
+
# dimensions' columns
|
78
|
+
'Time'=> { 'out_type' => 'timestamp', 'format' => "%Y-%m-%dT%H", 'timezone' => 'UTC' },
|
79
|
+
'Account'=> { 'out_type' => 'string' },
|
80
|
+
'Supply Source ID'=> { 'out_type' => 'string' },
|
81
|
+
'Supply Source'=> { 'out_type' => 'string' },
|
82
|
+
'Supply Partner'=> { 'out_type' => 'string' },
|
83
|
+
'Environment'=> { 'out_type' => 'string' },
|
84
|
+
'Domain'=> { 'out_type' => 'string' },
|
85
|
+
'App Name'=> { 'out_type' => 'string' },
|
86
|
+
'Bundle ID'=> { 'out_type' => 'string' },
|
87
|
+
'Supply Tag Type'=> { 'out_type' => 'string' },
|
88
|
+
'Demand Partner'=> { 'out_type' => 'string' },
|
89
|
+
'Demand Deal ID' => { 'out_type' => 'string' },
|
90
|
+
'Demand Deal'=> { 'out_type' => 'string' },
|
91
|
+
'Demand Tag'=> { 'out_type' => 'string' },
|
92
|
+
'Format'=> { 'out_type' => 'string' },
|
93
|
+
'Country'=> { 'out_type' => 'string' },
|
94
|
+
'Device Type'=> { 'out_type' => 'string' },
|
95
|
+
'OS'=> { 'out_type' => 'string' },
|
96
|
+
'Width X Height'=> { 'out_type' => 'string' },
|
97
|
+
# "Opportunity report" columns
|
98
|
+
'Tag Loads' => { 'out_type' => 'long' },
|
99
|
+
'Opportunities' => { 'out_type' => 'long' },
|
100
|
+
'Format Loads' => { 'out_type' => 'long' },
|
101
|
+
'Format Fill Rate' => { 'out_type' => 'double' },
|
102
|
+
'Ineligible Ops: Demand' => { 'out_type' => 'long' },
|
103
|
+
'Ineligible Ops: Restrictions' => { 'out_type' => 'long' },
|
104
|
+
'Impressions' => { 'out_type' => 'long' },
|
105
|
+
'Fill Rate' => { 'out_type' => 'double' },
|
106
|
+
'Efficiency Rate' => { 'out_type' => 'double' },
|
107
|
+
'CPM' => { 'out_type' => 'double' },
|
108
|
+
'Revenue' => { 'out_type' => 'double' },
|
109
|
+
'Cost' => { 'out_type' => 'double' },
|
110
|
+
'Profit' => { 'out_type' => 'double' },
|
111
|
+
'Profit Margin' => { 'out_type' => 'double' },
|
112
|
+
'Clicks' => { 'out_type' => 'long' },
|
113
|
+
'CTR' => { 'out_type' => 'double' },
|
114
|
+
'25% Views' => { 'out_type' => 'long' },
|
115
|
+
'50% Views' => { 'out_type' => 'long' },
|
116
|
+
'75% Views' => { 'out_type' => 'long' },
|
117
|
+
'100% Views' => { 'out_type' => 'long' },
|
118
|
+
'25% View Rate' => { 'out_type' => 'double' },
|
119
|
+
'50% View Rate' => { 'out_type' => 'double' },
|
120
|
+
'75% View Rate' => { 'out_type' => 'double' },
|
121
|
+
'100% View Rate' => { 'out_type' => 'double' },
|
122
|
+
'Viewability Measured Rate' => { 'out_type' => 'double' },
|
123
|
+
'Viewability Rate' => { 'out_type' => 'double' },
|
124
|
+
# "Request report" columns
|
125
|
+
'Tag Requests' => { 'out_type' => 'long' },
|
126
|
+
'Ads' => { 'out_type' => 'long' },
|
127
|
+
'VAST Ads' => { 'out_type' => 'long' },
|
128
|
+
'VPAID Ads' => { 'out_type' => 'long' },
|
129
|
+
'Wins' => { 'out_type' => 'long' },
|
130
|
+
'Ad Rate' => { 'out_type' => 'double' },
|
131
|
+
'VAST Ad Rate' => { 'out_type' => 'double' },
|
132
|
+
'VPAID Ad Rate' => { 'out_type' => 'double' },
|
133
|
+
'Win Rate' => { 'out_type' => 'double' },
|
134
|
+
'VPAID Responses' => { 'out_type' => 'long' },
|
135
|
+
'VPAID Attempts' => { 'out_type' => 'long' },
|
136
|
+
'VPAID Successes' => { 'out_type' => 'long' },
|
137
|
+
'VPAID Opt Outs' => { 'out_type' => 'long' },
|
138
|
+
'VPAID Timeouts' => { 'out_type' => 'long' },
|
139
|
+
'VPAID Errors' => { 'out_type' => 'long' },
|
140
|
+
'VPAID Success Rate' => { 'out_type' => 'double' },
|
141
|
+
'VPAID Opt Out Rate' => { 'out_type' => 'double' },
|
142
|
+
'VPAID Timeout Rate' => { 'out_type' => 'double' },
|
143
|
+
'VPAID Error Rate' => { 'out_type' => 'double' },
|
144
|
+
'Tag Timeouts' => { 'out_type' => 'long' },
|
145
|
+
'Tag Timeout Rate' => { 'out_type' => 'double' },
|
146
|
+
'Tag Errors' => { 'out_type' => 'long' },
|
147
|
+
'Tag Error Rate' => { 'out_type' => 'long' },
|
148
|
+
'Playback Errors' => { 'out_type' => 'long' },
|
149
|
+
'Playback Error Rate' => { 'out_type' => 'double' },
|
150
|
+
# custom data columns
|
151
|
+
'Custom 1'=> { 'out_type' => 'string' },
|
152
|
+
'Custom 2'=> { 'out_type' => 'string' },
|
153
|
+
'Custom 3'=> { 'out_type' => 'string' },
|
154
|
+
'Custom 4'=> { 'out_type' => 'string' },
|
155
|
+
'Custom 5'=> { 'out_type' => 'string' },
|
156
|
+
'Custom 6'=> { 'out_type' => 'string' },
|
157
|
+
'Custom 7'=> { 'out_type' => 'string' },
|
158
|
+
'Custom 8'=> { 'out_type' => 'string' },
|
159
|
+
'Custom 9'=> { 'out_type' => 'string' },
|
160
|
+
'Custom 10'=> { 'out_type' => 'string' },
|
161
|
+
}.freeze
|
56
162
|
end
|
57
163
|
|
58
164
|
end
|
59
165
|
end
|
166
|
+
|
metadata
CHANGED
@@ -1,52 +1,66 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-lkqd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ming Liu
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08-
|
11
|
+
date: 2017-08-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: embulk
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
|
-
- - "
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 2.2.2
|
19
|
+
name: http
|
20
|
+
prerelease: false
|
21
|
+
type: :runtime
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 2.2.2
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - '='
|
18
31
|
- !ruby/object:Gem::Version
|
19
32
|
version: 0.8.28
|
20
|
-
|
33
|
+
name: embulk
|
21
34
|
prerelease: false
|
35
|
+
type: :development
|
22
36
|
version_requirements: !ruby/object:Gem::Requirement
|
23
37
|
requirements:
|
24
|
-
- -
|
38
|
+
- - '='
|
25
39
|
- !ruby/object:Gem::Version
|
26
40
|
version: 0.8.28
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
|
-
name: bundler
|
29
42
|
requirement: !ruby/object:Gem::Requirement
|
30
43
|
requirements:
|
31
44
|
- - ">="
|
32
45
|
- !ruby/object:Gem::Version
|
33
46
|
version: 1.10.6
|
34
|
-
|
47
|
+
name: bundler
|
35
48
|
prerelease: false
|
49
|
+
type: :development
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
37
51
|
requirements:
|
38
52
|
- - ">="
|
39
53
|
- !ruby/object:Gem::Version
|
40
54
|
version: 1.10.6
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
|
-
name: rake
|
43
56
|
requirement: !ruby/object:Gem::Requirement
|
44
57
|
requirements:
|
45
58
|
- - ">="
|
46
59
|
- !ruby/object:Gem::Version
|
47
60
|
version: '10.0'
|
48
|
-
|
61
|
+
name: rake
|
49
62
|
prerelease: false
|
63
|
+
type: :development
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
66
|
- - ">="
|
@@ -67,11 +81,11 @@ files:
|
|
67
81
|
- Rakefile
|
68
82
|
- embulk-input-lkqd.gemspec
|
69
83
|
- lib/embulk/input/lkqd.rb
|
70
|
-
homepage:
|
84
|
+
homepage: https://github.com/liuming/embulk-input-lkqd
|
71
85
|
licenses:
|
72
86
|
- MIT
|
73
87
|
metadata: {}
|
74
|
-
post_install_message:
|
88
|
+
post_install_message:
|
75
89
|
rdoc_options: []
|
76
90
|
require_paths:
|
77
91
|
- lib
|
@@ -86,9 +100,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
86
100
|
- !ruby/object:Gem::Version
|
87
101
|
version: '0'
|
88
102
|
requirements: []
|
89
|
-
rubyforge_project:
|
103
|
+
rubyforge_project:
|
90
104
|
rubygems_version: 2.6.8
|
91
|
-
signing_key:
|
105
|
+
signing_key:
|
92
106
|
specification_version: 4
|
93
107
|
summary: LKQD input plugin for Embulk
|
94
108
|
test_files: []
|