embulk-input-lkqd 0.0.0 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +20 -9
  3. data/embulk-input-lkqd.gemspec +5 -3
  4. data/lib/embulk/input/lkqd.rb +131 -24
  5. metadata +29 -15
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dffec9445784f4667ccb1e7c3e9b55667102fdaa
4
- data.tar.gz: 9ff4d2c48b900585e887b1f9bb30fcf9e3f5e4ee
3
+ metadata.gz: b93579065f9e23f8657ba1347106a632be11d206
4
+ data.tar.gz: 68a43225355600b5f32a5cc747c901131615c70b
5
5
  SHA512:
6
- metadata.gz: b01226178d7293fbc8486120e14c6a17775e1ce3e71df00691d0a7c1c791157f1709a57239bb674494c358939bb3a7730135e296a81cb0b81207396d68bb62b7
7
- data.tar.gz: f92899f48bdf0e2347da43087075bc51f7f3d9641846c25c0e30a5cb1d180da4d8fa66d46447f28dc65ff0a61463b41deb259e4cc3cc5d83dc3638737ac41229
6
+ metadata.gz: 76e8dd612eb8ef05230b14dfff0a6c6af4d512cc9555a7429613fdbc01d217004efe06266eb1afe44167a6e63673080fcd3f9e82dd17bb9b59d2a507a5d49e25
7
+ data.tar.gz: 792d5f855f244a9c36752991c912f66862b90b41d69f9bbd655f21558c489f95b6cd4f2c55ce04ae81725dec96a11ea7cfdd141771f90aa1de1fe0f57fa4a5ef
data/README.md CHANGED
@@ -1,27 +1,38 @@
1
1
  # Lkqd input plugin for Embulk
2
2
 
3
- TODO: Write short description here and embulk-input-lkqd.gemspec file.
3
+ Loads reporting data from LKQD API.
4
4
 
5
5
  ## Overview
6
6
 
7
7
  * **Plugin type**: input
8
- * **Resume supported**: yes
9
- * **Cleanup supported**: yes
10
- * **Guess supported**: no
8
+ * **Resume supported**: no
9
+ * **Cleanup supported**: no
10
+ * **Guess supported**: yes
11
11
 
12
12
  ## Configuration
13
13
 
14
- - **option1**: description (integer, required)
15
- - **option2**: description (string, default: `"myvalue"`)
16
- - **option3**: description (string, default: `null`)
14
+ - **secret_key_id**: API Secret Key ID (string, required)
15
+ - **secret_key**: API Secret Key (string, required)
16
+ - **endpoint**: API endpoint (string, default: `'https://api.lkqd.com/reports'`)
17
+ - **report_parameters**: Report parameters documented in https://wiki.lkqd.com/display/API/LKQD+API (hash, default: `{}`)
17
18
 
18
19
  ## Example
19
20
 
20
21
  ```yaml
21
22
  in:
22
23
  type: lkqd
23
- option1: example1
24
- option2: example2
24
+ secret_key_id:
25
+ secret_key:
26
+ report_parameters:
27
+ timeDimension: "HOURLY"
28
+ reportType: [""]
29
+ reportFormat: "CSV"
30
+ startDate: "2017-08-01"
31
+ endDate: "2017-08-02"
32
+ startHour: 0
33
+ endHour: 23
34
+ timezone: "UTC"
35
+ metrics: [""]
25
36
  ```
26
37
 
27
38
 
data/embulk-input-lkqd.gemspec CHANGED
@@ -1,19 +1,21 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-input-lkqd"
3
- spec.version = "0.0.0"
3
+ spec.version = "0.1.0"
4
4
  spec.authors = ["Ming Liu"]
5
5
  spec.summary = "LKQD input plugin for Embulk"
6
6
  spec.description = "Loads reporting data from LKQD API."
7
7
  spec.email = ["liuming@lmws.net"]
8
8
  spec.licenses = ["MIT"]
9
- # TODO set this: spec.homepage = "https://github.com//embulk-input-lkqd"
9
+ spec.homepage = "https://github.com/liuming/embulk-input-lkqd"
10
10
 
11
11
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
12
12
  spec.test_files = spec.files.grep(%r{^(test|spec)/})
13
13
  spec.require_paths = ["lib"]
14
14
 
15
+ spec.add_dependency 'http', ['~> 2.2.2']
16
+
15
17
  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
16
- spec.add_development_dependency 'embulk', ['>= 0.8.28']
18
+ spec.add_development_dependency 'embulk', ['= 0.8.28']
17
19
  spec.add_development_dependency 'bundler', ['>= 1.10.6']
18
20
  spec.add_development_dependency 'rake', ['>= 10.0']
19
21
  end
data/lib/embulk/input/lkqd.rb CHANGED
@@ -1,3 +1,9 @@
1
+ require 'base64'
2
+ require 'json'
3
+ require 'csv'
4
+ require 'tempfile'
5
+ require 'http'
6
+
1
7
  module Embulk
2
8
  module Input
3
9
 
@@ -5,18 +11,25 @@ module Embulk
5
11
  Plugin.register_input("lkqd", self)
6
12
 
7
13
  def self.transaction(config, &control)
8
- # configuration code:
9
14
  task = {
10
- "option1" => config.param("option1", :integer), # integer, required
11
- "option2" => config.param("option2", :string, default: "myvalue"), # string, optional
12
- "option3" => config.param("option3", :string, default: nil), # string, optional
15
+ "secret_key_id" => config.param("secret_key_id", :string),
16
+ "secret_key" => config.param("secret_key", :string),
17
+ "endpoint" => config.param("endpoint", :string, default: 'https://api.lkqd.com/reports'),
18
+ "report_parameters" => config.param("report_parameters", :hash, default: {}),
13
19
  }
20
+ task['authorization'] = Base64.urlsafe_encode64("#{task['secret_key_id']}:#{task['secret_key']}")
21
+
22
+ response = request_lkqd({authorization: task['authorization'], endpoint: task['endpoint'], report_parameters: task['report_parameters']})
23
+ tempfile = Tempfile.new('lkqd_')
24
+ tempfile.write response.body
25
+ task['tempfile_path'] = tempfile.path
26
+ tempfile.close
14
27
 
15
- columns = [
16
- Column.new(0, "example", :string),
17
- Column.new(1, "column", :long),
18
- Column.new(2, "value", :double),
19
- ]
28
+ columns = []
29
+ ::CSV.foreach(tempfile.path).first.each_with_index do |column_name, index|
30
+ column_type = guess_column_type(task['report_parameters']['metrics'], column_name)
31
+ columns << Column.new({index: index}.merge(column_type))
32
+ end
20
33
 
21
34
  resume(task, columns, 1, &control)
22
35
  end
@@ -28,32 +41,126 @@ module Embulk
28
41
  return next_config_diff
29
42
  end
30
43
 
31
- # TODO
32
- # def self.guess(config)
33
- # sample_records = [
34
- # {"example"=>"a", "column"=>1, "value"=>0.1},
35
- # {"example"=>"a", "column"=>2, "value"=>0.2},
36
- # ]
37
- # columns = Guess::SchemaGuess.from_hash_records(sample_records)
38
- # return {"columns" => columns}
39
- # end
44
+ def self.guess(config)
45
+ return {}
46
+ end
47
+
48
+ def self.request_lkqd(config)
49
+ ::HTTP.auth("Basic #{config[:authorization]}").post(config[:endpoint], json: config[:report_parameters])
50
+ end
51
+
52
+ def self.guess_column_type(metrics, column_name)
53
+ column_name.gsub!(/^\W/, '')
54
+ column_option = DEFAULT_COLUMNS[column_name]
55
+ if column_option
56
+ return {type: column_option['out_type'].to_sym, name: column_name, format: column_option['format']}
57
+ else
58
+ return {type: :string, name: column_name}
59
+ end
60
+ end
40
61
 
41
62
  def init
42
- # initialization code:
43
- @option1 = task["option1"]
44
- @option2 = task["option2"]
45
- @option3 = task["option3"]
63
+ @task = task
46
64
  end
47
65
 
48
66
  def run
49
- page_builder.add(["example-value", 1, 0.1])
50
- page_builder.add(["example-value", 2, 0.2])
67
+ CSV.foreach(@task['tempfile_path'], {headers: true}).each do |row|
68
+ page_builder.add(row)
69
+ end
51
70
  page_builder.finish
52
71
 
53
72
  task_report = {}
54
73
  return task_report
55
74
  end
75
+
76
+ DEFAULT_COLUMNS = {
77
+ # dimensions' columns
78
+ 'Time'=> { 'out_type' => 'timestamp', 'format' => "%Y-%m-%dT%H", 'timezone' => 'UTC' },
79
+ 'Account'=> { 'out_type' => 'string' },
80
+ 'Supply Source ID'=> { 'out_type' => 'string' },
81
+ 'Supply Source'=> { 'out_type' => 'string' },
82
+ 'Supply Partner'=> { 'out_type' => 'string' },
83
+ 'Environment'=> { 'out_type' => 'string' },
84
+ 'Domain'=> { 'out_type' => 'string' },
85
+ 'App Name'=> { 'out_type' => 'string' },
86
+ 'Bundle ID'=> { 'out_type' => 'string' },
87
+ 'Supply Tag Type'=> { 'out_type' => 'string' },
88
+ 'Demand Partner'=> { 'out_type' => 'string' },
89
+ 'Demand Deal ID' => { 'out_type' => 'string' },
90
+ 'Demand Deal'=> { 'out_type' => 'string' },
91
+ 'Demand Tag'=> { 'out_type' => 'string' },
92
+ 'Format'=> { 'out_type' => 'string' },
93
+ 'Country'=> { 'out_type' => 'string' },
94
+ 'Device Type'=> { 'out_type' => 'string' },
95
+ 'OS'=> { 'out_type' => 'string' },
96
+ 'Width X Height'=> { 'out_type' => 'string' },
97
+ # "Opportunity report" columns
98
+ 'Tag Loads' => { 'out_type' => 'long' },
99
+ 'Opportunities' => { 'out_type' => 'long' },
100
+ 'Format Loads' => { 'out_type' => 'long' },
101
+ 'Format Fill Rate' => { 'out_type' => 'double' },
102
+ 'Ineligible Ops: Demand' => { 'out_type' => 'long' },
103
+ 'Ineligible Ops: Restrictions' => { 'out_type' => 'long' },
104
+ 'Impressions' => { 'out_type' => 'long' },
105
+ 'Fill Rate' => { 'out_type' => 'double' },
106
+ 'Efficiency Rate' => { 'out_type' => 'double' },
107
+ 'CPM' => { 'out_type' => 'double' },
108
+ 'Revenue' => { 'out_type' => 'double' },
109
+ 'Cost' => { 'out_type' => 'double' },
110
+ 'Profit' => { 'out_type' => 'double' },
111
+ 'Profit Margin' => { 'out_type' => 'double' },
112
+ 'Clicks' => { 'out_type' => 'long' },
113
+ 'CTR' => { 'out_type' => 'double' },
114
+ '25% Views' => { 'out_type' => 'long' },
115
+ '50% Views' => { 'out_type' => 'long' },
116
+ '75% Views' => { 'out_type' => 'long' },
117
+ '100% Views' => { 'out_type' => 'long' },
118
+ '25% View Rate' => { 'out_type' => 'double' },
119
+ '50% View Rate' => { 'out_type' => 'double' },
120
+ '75% View Rate' => { 'out_type' => 'double' },
121
+ '100% View Rate' => { 'out_type' => 'double' },
122
+ 'Viewability Measured Rate' => { 'out_type' => 'double' },
123
+ 'Viewability Rate' => { 'out_type' => 'double' },
124
+ # "Request report" columns
125
+ 'Tag Requests' => { 'out_type' => 'long' },
126
+ 'Ads' => { 'out_type' => 'long' },
127
+ 'VAST Ads' => { 'out_type' => 'long' },
128
+ 'VPAID Ads' => { 'out_type' => 'long' },
129
+ 'Wins' => { 'out_type' => 'long' },
130
+ 'Ad Rate' => { 'out_type' => 'double' },
131
+ 'VAST Ad Rate' => { 'out_type' => 'double' },
132
+ 'VPAID Ad Rate' => { 'out_type' => 'double' },
133
+ 'Win Rate' => { 'out_type' => 'double' },
134
+ 'VPAID Responses' => { 'out_type' => 'long' },
135
+ 'VPAID Attempts' => { 'out_type' => 'long' },
136
+ 'VPAID Successes' => { 'out_type' => 'long' },
137
+ 'VPAID Opt Outs' => { 'out_type' => 'long' },
138
+ 'VPAID Timeouts' => { 'out_type' => 'long' },
139
+ 'VPAID Errors' => { 'out_type' => 'long' },
140
+ 'VPAID Success Rate' => { 'out_type' => 'double' },
141
+ 'VPAID Opt Out Rate' => { 'out_type' => 'double' },
142
+ 'VPAID Timeout Rate' => { 'out_type' => 'double' },
143
+ 'VPAID Error Rate' => { 'out_type' => 'double' },
144
+ 'Tag Timeouts' => { 'out_type' => 'long' },
145
+ 'Tag Timeout Rate' => { 'out_type' => 'double' },
146
+ 'Tag Errors' => { 'out_type' => 'long' },
147
+ 'Tag Error Rate' => { 'out_type' => 'long' },
148
+ 'Playback Errors' => { 'out_type' => 'long' },
149
+ 'Playback Error Rate' => { 'out_type' => 'double' },
150
+ # custom data columns
151
+ 'Custom 1'=> { 'out_type' => 'string' },
152
+ 'Custom 2'=> { 'out_type' => 'string' },
153
+ 'Custom 3'=> { 'out_type' => 'string' },
154
+ 'Custom 4'=> { 'out_type' => 'string' },
155
+ 'Custom 5'=> { 'out_type' => 'string' },
156
+ 'Custom 6'=> { 'out_type' => 'string' },
157
+ 'Custom 7'=> { 'out_type' => 'string' },
158
+ 'Custom 8'=> { 'out_type' => 'string' },
159
+ 'Custom 9'=> { 'out_type' => 'string' },
160
+ 'Custom 10'=> { 'out_type' => 'string' },
161
+ }.freeze
56
162
  end
57
163
 
58
164
  end
59
165
  end
166
+
metadata CHANGED
@@ -1,52 +1,66 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-lkqd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ming Liu
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-03 00:00:00.000000000 Z
11
+ date: 2017-08-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: embulk
15
14
  requirement: !ruby/object:Gem::Requirement
16
15
  requirements:
17
- - - ">="
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: 2.2.2
19
+ name: http
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 2.2.2
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - '='
18
31
  - !ruby/object:Gem::Version
19
32
  version: 0.8.28
20
- type: :development
33
+ name: embulk
21
34
  prerelease: false
35
+ type: :development
22
36
  version_requirements: !ruby/object:Gem::Requirement
23
37
  requirements:
24
- - - ">="
38
+ - - '='
25
39
  - !ruby/object:Gem::Version
26
40
  version: 0.8.28
27
41
  - !ruby/object:Gem::Dependency
28
- name: bundler
29
42
  requirement: !ruby/object:Gem::Requirement
30
43
  requirements:
31
44
  - - ">="
32
45
  - !ruby/object:Gem::Version
33
46
  version: 1.10.6
34
- type: :development
47
+ name: bundler
35
48
  prerelease: false
49
+ type: :development
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ">="
39
53
  - !ruby/object:Gem::Version
40
54
  version: 1.10.6
41
55
  - !ruby/object:Gem::Dependency
42
- name: rake
43
56
  requirement: !ruby/object:Gem::Requirement
44
57
  requirements:
45
58
  - - ">="
46
59
  - !ruby/object:Gem::Version
47
60
  version: '10.0'
48
- type: :development
61
+ name: rake
49
62
  prerelease: false
63
+ type: :development
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - ">="
@@ -67,11 +81,11 @@ files:
67
81
  - Rakefile
68
82
  - embulk-input-lkqd.gemspec
69
83
  - lib/embulk/input/lkqd.rb
70
- homepage:
84
+ homepage: https://github.com/liuming/embulk-input-lkqd
71
85
  licenses:
72
86
  - MIT
73
87
  metadata: {}
74
- post_install_message:
88
+ post_install_message:
75
89
  rdoc_options: []
76
90
  require_paths:
77
91
  - lib
@@ -86,9 +100,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
86
100
  - !ruby/object:Gem::Version
87
101
  version: '0'
88
102
  requirements: []
89
- rubyforge_project:
103
+ rubyforge_project:
90
104
  rubygems_version: 2.6.8
91
- signing_key:
105
+ signing_key:
92
106
  specification_version: 4
93
107
  summary: LKQD input plugin for Embulk
94
108
  test_files: []