embulk-input-lkqd 0.0.0 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +20 -9
  3. data/embulk-input-lkqd.gemspec +5 -3
  4. data/lib/embulk/input/lkqd.rb +131 -24
  5. metadata +29 -15
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dffec9445784f4667ccb1e7c3e9b55667102fdaa
4
- data.tar.gz: 9ff4d2c48b900585e887b1f9bb30fcf9e3f5e4ee
3
+ metadata.gz: b93579065f9e23f8657ba1347106a632be11d206
4
+ data.tar.gz: 68a43225355600b5f32a5cc747c901131615c70b
5
5
  SHA512:
6
- metadata.gz: b01226178d7293fbc8486120e14c6a17775e1ce3e71df00691d0a7c1c791157f1709a57239bb674494c358939bb3a7730135e296a81cb0b81207396d68bb62b7
7
- data.tar.gz: f92899f48bdf0e2347da43087075bc51f7f3d9641846c25c0e30a5cb1d180da4d8fa66d46447f28dc65ff0a61463b41deb259e4cc3cc5d83dc3638737ac41229
6
+ metadata.gz: 76e8dd612eb8ef05230b14dfff0a6c6af4d512cc9555a7429613fdbc01d217004efe06266eb1afe44167a6e63673080fcd3f9e82dd17bb9b59d2a507a5d49e25
7
+ data.tar.gz: 792d5f855f244a9c36752991c912f66862b90b41d69f9bbd655f21558c489f95b6cd4f2c55ce04ae81725dec96a11ea7cfdd141771f90aa1de1fe0f57fa4a5ef
data/README.md CHANGED
@@ -1,27 +1,38 @@
1
1
  # Lkqd input plugin for Embulk
2
2
 
3
- TODO: Write short description here and embulk-input-lkqd.gemspec file.
3
+ Loads reporting data from LKQD API.
4
4
 
5
5
  ## Overview
6
6
 
7
7
  * **Plugin type**: input
8
- * **Resume supported**: yes
9
- * **Cleanup supported**: yes
10
- * **Guess supported**: no
8
+ * **Resume supported**: no
9
+ * **Cleanup supported**: no
10
+ * **Guess supported**: yes
11
11
 
12
12
  ## Configuration
13
13
 
14
- - **option1**: description (integer, required)
15
- - **option2**: description (string, default: `"myvalue"`)
16
- - **option3**: description (string, default: `null`)
14
+ - **secret_key_id**: API Secret Key ID (string, required)
15
+ - **secret_key**: API Secret Key (string, required)
16
+ - **endpoint**: API endpoint (string, default: `'https://api.lkqd.com/reports'`)
17
+ - **report_parameters**: Report parameters documented in https://wiki.lkqd.com/display/API/LKQD+API (hash, default: `{}`)
17
18
 
18
19
  ## Example
19
20
 
20
21
  ```yaml
21
22
  in:
22
23
  type: lkqd
23
- option1: example1
24
- option2: example2
24
+ secret_key_id:
25
+ secret_key:
26
+ report_parameters:
27
+ timeDimension: "HOURLY"
28
+ reportType: [""]
29
+ reportFormat: "CSV"
30
+ startDate: "2017-08-01"
31
+ endDate: "2017-08-02"
32
+ startHour: 0
33
+ endHour: 23
34
+ timezone: "UTC"
35
+ metrics: [""]
25
36
  ```
26
37
 
27
38
 
data/embulk-input-lkqd.gemspec CHANGED
@@ -1,19 +1,21 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-input-lkqd"
3
- spec.version = "0.0.0"
3
+ spec.version = "0.1.0"
4
4
  spec.authors = ["Ming Liu"]
5
5
  spec.summary = "LKQD input plugin for Embulk"
6
6
  spec.description = "Loads reporting data from LKQD API."
7
7
  spec.email = ["liuming@lmws.net"]
8
8
  spec.licenses = ["MIT"]
9
- # TODO set this: spec.homepage = "https://github.com//embulk-input-lkqd"
9
+ spec.homepage = "https://github.com/liuming/embulk-input-lkqd"
10
10
 
11
11
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
12
12
  spec.test_files = spec.files.grep(%r{^(test|spec)/})
13
13
  spec.require_paths = ["lib"]
14
14
 
15
+ spec.add_dependency 'http', ['~> 2.2.2']
16
+
15
17
  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
16
- spec.add_development_dependency 'embulk', ['>= 0.8.28']
18
+ spec.add_development_dependency 'embulk', ['= 0.8.28']
17
19
  spec.add_development_dependency 'bundler', ['>= 1.10.6']
18
20
  spec.add_development_dependency 'rake', ['>= 10.0']
19
21
  end
data/lib/embulk/input/lkqd.rb CHANGED
@@ -1,3 +1,9 @@
1
+ require 'base64'
2
+ require 'json'
3
+ require 'csv'
4
+ require 'tempfile'
5
+ require 'http'
6
+
1
7
  module Embulk
2
8
  module Input
3
9
 
@@ -5,18 +11,25 @@ module Embulk
5
11
  Plugin.register_input("lkqd", self)
6
12
 
7
13
  def self.transaction(config, &control)
8
- # configuration code:
9
14
  task = {
10
- "option1" => config.param("option1", :integer), # integer, required
11
- "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
- "option3" => config.param("option3", :string, default: nil), # string, optional
15
+ "secret_key_id" => config.param("secret_key_id", :string),
16
+ "secret_key" => config.param("secret_key", :string),
17
+ "endpoint" => config.param("endpoint", :string, default: 'https://api.lkqd.com/reports'),
18
+ "report_parameters" => config.param("report_parameters", :hash, default: {}),
13
19
  }
20
+ task['authorization'] = Base64.urlsafe_encode64("#{task['secret_key_id']}:#{task['secret_key']}")
21
+
22
+ response = request_lkqd({authorization: task['authorization'], endpoint: task['endpoint'], report_parameters: task['report_parameters']})
23
+ tempfile = Tempfile.new('lkqd_')
24
+ tempfile.write response.body
25
+ task['tempfile_path'] = tempfile.path
26
+ tempfile.close
14
27
 
15
- columns = [
16
- Column.new(0, "example", :string),
17
- Column.new(1, "column", :long),
18
- Column.new(2, "value", :double),
19
- ]
28
+ columns = []
29
+ ::CSV.foreach(tempfile.path).first.each_with_index do |column_name, index|
30
+ column_type = guess_column_type(task['report_parameters']['metrics'], column_name)
31
+ columns << Column.new({index: index}.merge(column_type))
32
+ end
20
33
 
21
34
  resume(task, columns, 1, &control)
22
35
  end
@@ -28,32 +41,126 @@ module Embulk
28
41
  return next_config_diff
29
42
  end
30
43
 
31
- # TODO
32
- # def self.guess(config)
33
- # sample_records = [
34
- # {"example"=>"a", "column"=>1, "value"=>0.1},
35
- # {"example"=>"a", "column"=>2, "value"=>0.2},
36
- # ]
37
- # columns = Guess::SchemaGuess.from_hash_records(sample_records)
38
- # return {"columns" => columns}
39
- # end
44
+ def self.guess(config)
45
+ return {}
46
+ end
47
+
48
+ def self.request_lkqd(config)
49
+ ::HTTP.auth("Basic #{config[:authorization]}").post(config[:endpoint], json: config[:report_parameters])
50
+ end
51
+
52
+ def self.guess_column_type(metrics, column_name)
53
+ column_name.gsub!(/^\W/, '')
54
+ column_option = DEFAULT_COLUMNS[column_name]
55
+ if column_option
56
+ return {type: column_option['out_type'].to_sym, name: column_name, format: column_option['format']}
57
+ else
58
+ return {type: :string, name: column_name}
59
+ end
60
+ end
40
61
 
41
62
  def init
42
- # initialization code:
43
- @option1 = task["option1"]
44
- @option2 = task["option2"]
45
- @option3 = task["option3"]
63
+ @task = task
46
64
  end
47
65
 
48
66
  def run
49
- page_builder.add(["example-value", 1, 0.1])
50
- page_builder.add(["example-value", 2, 0.2])
67
+ CSV.foreach(@task['tempfile_path'], {headers: true}).each do |row|
68
+ page_builder.add(row)
69
+ end
51
70
  page_builder.finish
52
71
 
53
72
  task_report = {}
54
73
  return task_report
55
74
  end
75
+
76
+ DEFAULT_COLUMNS = {
77
+ # dimensions' columns
78
+ 'Time'=> { 'out_type' => 'timestamp', 'format' => "%Y-%m-%dT%H", 'timezone' => 'UTC' },
79
+ 'Account'=> { 'out_type' => 'string' },
80
+ 'Supply Source ID'=> { 'out_type' => 'string' },
81
+ 'Supply Source'=> { 'out_type' => 'string' },
82
+ 'Supply Partner'=> { 'out_type' => 'string' },
83
+ 'Environment'=> { 'out_type' => 'string' },
84
+ 'Domain'=> { 'out_type' => 'string' },
85
+ 'App Name'=> { 'out_type' => 'string' },
86
+ 'Bundle ID'=> { 'out_type' => 'string' },
87
+ 'Supply Tag Type'=> { 'out_type' => 'string' },
88
+ 'Demand Partner'=> { 'out_type' => 'string' },
89
+ 'Demand Deal ID' => { 'out_type' => 'string' },
90
+ 'Demand Deal'=> { 'out_type' => 'string' },
91
+ 'Demand Tag'=> { 'out_type' => 'string' },
92
+ 'Format'=> { 'out_type' => 'string' },
93
+ 'Country'=> { 'out_type' => 'string' },
94
+ 'Device Type'=> { 'out_type' => 'string' },
95
+ 'OS'=> { 'out_type' => 'string' },
96
+ 'Width X Height'=> { 'out_type' => 'string' },
97
+ # "Opportunity report" columns
98
+ 'Tag Loads' => { 'out_type' => 'long' },
99
+ 'Opportunities' => { 'out_type' => 'long' },
100
+ 'Format Loads' => { 'out_type' => 'long' },
101
+ 'Format Fill Rate' => { 'out_type' => 'double' },
102
+ 'Ineligible Ops: Demand' => { 'out_type' => 'long' },
103
+ 'Ineligible Ops: Restrictions' => { 'out_type' => 'long' },
104
+ 'Impressions' => { 'out_type' => 'long' },
105
+ 'Fill Rate' => { 'out_type' => 'double' },
106
+ 'Efficiency Rate' => { 'out_type' => 'double' },
107
+ 'CPM' => { 'out_type' => 'double' },
108
+ 'Revenue' => { 'out_type' => 'double' },
109
+ 'Cost' => { 'out_type' => 'double' },
110
+ 'Profit' => { 'out_type' => 'double' },
111
+ 'Profit Margin' => { 'out_type' => 'double' },
112
+ 'Clicks' => { 'out_type' => 'long' },
113
+ 'CTR' => { 'out_type' => 'double' },
114
+ '25% Views' => { 'out_type' => 'long' },
115
+ '50% Views' => { 'out_type' => 'long' },
116
+ '75% Views' => { 'out_type' => 'long' },
117
+ '100% Views' => { 'out_type' => 'long' },
118
+ '25% View Rate' => { 'out_type' => 'double' },
119
+ '50% View Rate' => { 'out_type' => 'double' },
120
+ '75% View Rate' => { 'out_type' => 'double' },
121
+ '100% View Rate' => { 'out_type' => 'double' },
122
+ 'Viewability Measured Rate' => { 'out_type' => 'double' },
123
+ 'Viewability Rate' => { 'out_type' => 'double' },
124
+ # "Request report" columns
125
+ 'Tag Requests' => { 'out_type' => 'long' },
126
+ 'Ads' => { 'out_type' => 'long' },
127
+ 'VAST Ads' => { 'out_type' => 'long' },
128
+ 'VPAID Ads' => { 'out_type' => 'long' },
129
+ 'Wins' => { 'out_type' => 'long' },
130
+ 'Ad Rate' => { 'out_type' => 'double' },
131
+ 'VAST Ad Rate' => { 'out_type' => 'double' },
132
+ 'VPAID Ad Rate' => { 'out_type' => 'double' },
133
+ 'Win Rate' => { 'out_type' => 'double' },
134
+ 'VPAID Responses' => { 'out_type' => 'long' },
135
+ 'VPAID Attempts' => { 'out_type' => 'long' },
136
+ 'VPAID Successes' => { 'out_type' => 'long' },
137
+ 'VPAID Opt Outs' => { 'out_type' => 'long' },
138
+ 'VPAID Timeouts' => { 'out_type' => 'long' },
139
+ 'VPAID Errors' => { 'out_type' => 'long' },
140
+ 'VPAID Success Rate' => { 'out_type' => 'double' },
141
+ 'VPAID Opt Out Rate' => { 'out_type' => 'double' },
142
+ 'VPAID Timeout Rate' => { 'out_type' => 'double' },
143
+ 'VPAID Error Rate' => { 'out_type' => 'double' },
144
+ 'Tag Timeouts' => { 'out_type' => 'long' },
145
+ 'Tag Timeout Rate' => { 'out_type' => 'double' },
146
+ 'Tag Errors' => { 'out_type' => 'long' },
147
+ 'Tag Error Rate' => { 'out_type' => 'long' },
148
+ 'Playback Errors' => { 'out_type' => 'long' },
149
+ 'Playback Error Rate' => { 'out_type' => 'double' },
150
+ # custom data columns
151
+ 'Custom 1'=> { 'out_type' => 'string' },
152
+ 'Custom 2'=> { 'out_type' => 'string' },
153
+ 'Custom 3'=> { 'out_type' => 'string' },
154
+ 'Custom 4'=> { 'out_type' => 'string' },
155
+ 'Custom 5'=> { 'out_type' => 'string' },
156
+ 'Custom 6'=> { 'out_type' => 'string' },
157
+ 'Custom 7'=> { 'out_type' => 'string' },
158
+ 'Custom 8'=> { 'out_type' => 'string' },
159
+ 'Custom 9'=> { 'out_type' => 'string' },
160
+ 'Custom 10'=> { 'out_type' => 'string' },
161
+ }.freeze
56
162
  end
57
163
 
58
164
  end
59
165
  end
166
+
metadata CHANGED
@@ -1,52 +1,66 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-lkqd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ming Liu
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-03 00:00:00.000000000 Z
11
+ date: 2017-08-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: embulk
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
- - - ">="
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: 2.2.2
19
+ name: http
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 2.2.2
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - '='
18
31
  - !ruby/object:Gem::Version
19
32
  version: 0.8.28
20
- type: :development
33
+ name: embulk
21
34
  prerelease: false
35
+ type: :development
22
36
  version_requirements: !ruby/object:Gem::Requirement
23
37
  requirements:
24
- - - ">="
38
+ - - '='
25
39
  - !ruby/object:Gem::Version
26
40
  version: 0.8.28
27
41
  - !ruby/object:Gem::Dependency
28
- name: bundler
29
42
  requirement: !ruby/object:Gem::Requirement
30
43
  requirements:
31
44
  - - ">="
32
45
  - !ruby/object:Gem::Version
33
46
  version: 1.10.6
34
- type: :development
47
+ name: bundler
35
48
  prerelease: false
49
+ type: :development
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ">="
39
53
  - !ruby/object:Gem::Version
40
54
  version: 1.10.6
41
55
  - !ruby/object:Gem::Dependency
42
- name: rake
43
56
  requirement: !ruby/object:Gem::Requirement
44
57
  requirements:
45
58
  - - ">="
46
59
  - !ruby/object:Gem::Version
47
60
  version: '10.0'
48
- type: :development
61
+ name: rake
49
62
  prerelease: false
63
+ type: :development
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - ">="
@@ -67,11 +81,11 @@ files:
67
81
  - Rakefile
68
82
  - embulk-input-lkqd.gemspec
69
83
  - lib/embulk/input/lkqd.rb
70
- homepage:
84
+ homepage: https://github.com/liuming/embulk-input-lkqd
71
85
  licenses:
72
86
  - MIT
73
87
  metadata: {}
74
- post_install_message:
88
+ post_install_message:
75
89
  rdoc_options: []
76
90
  require_paths:
77
91
  - lib
@@ -86,9 +100,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
86
100
  - !ruby/object:Gem::Version
87
101
  version: '0'
88
102
  requirements: []
89
- rubyforge_project:
103
+ rubyforge_project:
90
104
  rubygems_version: 2.6.8
91
- signing_key:
105
+ signing_key:
92
106
  specification_version: 4
93
107
  summary: LKQD input plugin for Embulk
94
108
  test_files: []