embulk-input-zendesk 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +5 -1
- data/Rakefile +1 -1
- data/embulk-input-zendesk.gemspec +1 -1
- data/lib/embulk/input/zendesk/client.rb +30 -19
- data/lib/embulk/input/zendesk/plugin.rb +25 -4
- data/test/embulk/input/zendesk/test_plugin.rb +167 -12
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ed2bcc09ae824662952c09511b06045ebef3641
|
4
|
+
data.tar.gz: 95281d1ac59e524c36356c9c670de71c067fb326
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4994efad0da2b8872ceda6c29a0cd5eda72136194a33e71c6e6a418dbf412f95c289463d243c7e812b30c23a56f79dee19c344461667b722e10231c44cdcdfb5
|
7
|
+
data.tar.gz: b9b63e1fde1f724cc418e39a529c9bf0f3ebbf831bfdd5f268ef3027b71b25fa7987f3370db058ce51c12e9e704f01a0bac55d40f26305104a3debf92fd60ab1
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.1.3 - 2016-03-15
|
2
|
+
|
3
|
+
* [enhancement] Support more targets [#8](https://github.com/treasure-data/embulk-input-zendesk/pull/8)
|
4
|
+
* [enhancement] Enable json type [#7](https://github.com/treasure-data/embulk-input-zendesk/pull/7)
|
5
|
+
|
1
6
|
## 0.1.2 - 2016-01-29
|
2
7
|
|
3
8
|
* [maintenance] Add authors @muga and @sakama.
|
data/README.md
CHANGED
@@ -22,7 +22,8 @@ Required Embulk version >= 0.8.1.
|
|
22
22
|
|
23
23
|
- **login_url**: Login URL for Zendesk (string, required)
|
24
24
|
- **auth_method**: `basic`, `token`, or `oauth`. For more detail on [zendesk document](https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication). (string, required)
|
25
|
-
- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields` or `
|
25
|
+
- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields`, `ticket_forms` or `ticket_metrics`. (string, required)
|
26
|
+
- **includes**: Will fetch sub resources. For example, ticket has ticket_audits, ticket_comments. See below example config. (array, default: `[]`)
|
26
27
|
- **username**: The user name a.k.a. email. Required if `auth_method` is `basic` or `token`. (string, default: `null`)
|
27
28
|
- **password**: Password. required if `auth_method` is `basic`. (string, default: `null`)
|
28
29
|
- **token**: Token. required if `auth_method` is `token`. (string, default: `null`)
|
@@ -42,6 +43,9 @@ in:
|
|
42
43
|
username: jdoe@example.com
|
43
44
|
token: 6wiIBWbGkBMo1mRDMuVwkw1EPsNkeUj95PIz2akv
|
44
45
|
target: tickets
|
46
|
+
includes:
|
47
|
+
- audits
|
48
|
+
- comments
|
45
49
|
start_time: "2015-01-01 00:00:00+0000"
|
46
50
|
```
|
47
51
|
|
data/Rakefile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-input-zendesk"
|
4
|
-
spec.version = "0.1.
|
4
|
+
spec.version = "0.1.3"
|
5
5
|
spec.authors = ["uu59", "muga", "sakama"]
|
6
6
|
spec.summary = "Zendesk input plugin for Embulk"
|
7
7
|
spec.description = "Loads records from Zendesk."
|
@@ -7,11 +7,9 @@ module Embulk
|
|
7
7
|
attr_reader :config
|
8
8
|
|
9
9
|
PARTIAL_RECORDS_SIZE = 50
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
).freeze
|
14
|
-
AVAILABLE_INCREMENTAL_EXPORT = AVAILABLE_TARGETS - %w(ticket_fields ticket_forms)
|
10
|
+
AVAILABLE_INCREMENTAL_EXPORT = %w(tickets users organizations ticket_events).freeze
|
11
|
+
UNAVAILABLE_INCREMENTAL_EXPORT = %w(ticket_fields ticket_forms ticket_metrics).freeze
|
12
|
+
AVAILABLE_TARGETS = AVAILABLE_INCREMENTAL_EXPORT + UNAVAILABLE_INCREMENTAL_EXPORT
|
15
13
|
|
16
14
|
def initialize(config)
|
17
15
|
@config = config
|
@@ -47,10 +45,11 @@ module Embulk
|
|
47
45
|
|
48
46
|
def validate_target
|
49
47
|
unless AVAILABLE_TARGETS.include?(config[:target])
|
50
|
-
raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported.")
|
48
|
+
raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported. Supported targets are #{AVAILABLE_TARGETS.join(", ")}.")
|
51
49
|
end
|
52
50
|
end
|
53
51
|
|
52
|
+
# they have both Incremental API and non-incremental API
|
54
53
|
%w(tickets users organizations).each do |target|
|
55
54
|
define_method(target) do |partial = true, start_time = 0, &block|
|
56
55
|
if partial
|
@@ -61,22 +60,32 @@ module Embulk
|
|
61
60
|
end
|
62
61
|
end
|
63
62
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
63
|
+
# they have incremental API only
|
64
|
+
%w(ticket_events).each do |target|
|
65
|
+
define_method(target) do |partial = true, start_time = 0, &block|
|
66
|
+
path = "/api/v2/incremental/#{target}"
|
67
|
+
incremental_export(path, target, start_time, [], &block)
|
68
|
+
end
|
68
69
|
end
|
69
70
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
71
|
+
# they have non-incremental API only
|
72
|
+
UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
|
73
|
+
define_method(target) do |partial = true, start_time = 0, &block|
|
74
|
+
path = "/api/v2/#{target}.json"
|
75
|
+
export(path, target, partial ? PARTIAL_RECORDS_SIZE : 1000, &block)
|
76
|
+
end
|
74
77
|
end
|
75
78
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
79
|
+
def fetch_subresource(record_id, base, target)
|
80
|
+
response = request("/api/v2/#{base}/#{record_id}/#{target}.json")
|
81
|
+
return [] if response.status == 404
|
82
|
+
|
83
|
+
begin
|
84
|
+
data = JSON.parse(response.body)
|
85
|
+
data[target]
|
86
|
+
rescue => e
|
87
|
+
raise Embulk::DataError.new(e)
|
88
|
+
end
|
80
89
|
end
|
81
90
|
|
82
91
|
private
|
@@ -163,12 +172,14 @@ module Embulk
|
|
163
172
|
u.path = path
|
164
173
|
|
165
174
|
retryer.with_retry do
|
175
|
+
Embulk.logger.debug "Fetching #{u.to_s}"
|
166
176
|
response = httpclient.get(u.to_s, query, follow_redirect: true)
|
167
177
|
|
168
178
|
# https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
|
169
179
|
status_code = response.status
|
170
180
|
case status_code
|
171
|
-
when 200
|
181
|
+
when 200, 404
|
182
|
+
# 404 would be returned e.g. ticket comments are empty (on fetch_subresource method)
|
172
183
|
response
|
173
184
|
when 400, 401
|
174
185
|
raise Embulk::ConfigError.new("[#{status_code}] #{response.body}")
|
@@ -58,12 +58,16 @@ module Embulk
|
|
58
58
|
hash[:type] = :json
|
59
59
|
end
|
60
60
|
|
61
|
-
# NOTE: current version don't support JSON type
|
62
|
-
next if hash[:type] == :json
|
63
|
-
|
64
61
|
hash
|
65
62
|
end
|
66
63
|
|
64
|
+
task[:includes].each do |ent|
|
65
|
+
columns << {
|
66
|
+
name: ent,
|
67
|
+
type: :json
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
67
71
|
return {"columns" => columns.compact}
|
68
72
|
end
|
69
73
|
|
@@ -81,6 +85,7 @@ module Embulk
|
|
81
85
|
retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
|
82
86
|
incremental: config.param("incremental", :bool, default: true),
|
83
87
|
schema: config.param(:columns, :array, default: []),
|
88
|
+
includes: config.param(:includes, :array, default: []),
|
84
89
|
}
|
85
90
|
end
|
86
91
|
|
@@ -95,8 +100,8 @@ module Embulk
|
|
95
100
|
args << @start_time.to_i
|
96
101
|
end
|
97
102
|
|
98
|
-
client = Client.new(task)
|
99
103
|
last_data = client.public_send(method, *args) do |record|
|
104
|
+
record = fetch_related_object(record)
|
100
105
|
values = extract_values(record)
|
101
106
|
page_builder.add(values)
|
102
107
|
end
|
@@ -116,6 +121,22 @@ module Embulk
|
|
116
121
|
|
117
122
|
private
|
118
123
|
|
124
|
+
def fetch_related_object(record)
|
125
|
+
(task[:includes] || []).each do |ent|
|
126
|
+
if preview?
|
127
|
+
# Fetching subresource consume ~2 sec for each record. it is too long to preview. so the dummy value used.
|
128
|
+
record[ent] = [{dummy: "(#{ent}) dummy value for preview"}]
|
129
|
+
else
|
130
|
+
record[ent] = client.fetch_subresource(record["id"], task[:target], ent)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
record
|
134
|
+
end
|
135
|
+
|
136
|
+
def client
|
137
|
+
Client.new(task)
|
138
|
+
end
|
139
|
+
|
119
140
|
def preview?
|
120
141
|
org.embulk.spi.Exec.isPreview()
|
121
142
|
rescue java.lang.NullPointerException => e
|
@@ -132,13 +132,162 @@ module Embulk
|
|
132
132
|
assert actual.include?(name: "id", type: :long)
|
133
133
|
assert actual.include?(name: "created_at", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z")
|
134
134
|
assert actual.include?(name: "has_incidents", type: :boolean)
|
135
|
+
assert actual.include?(name: "tags", type: :json)
|
136
|
+
assert actual.include?(name: "collaborator_ids", type: :json)
|
137
|
+
assert actual.include?(name: "custom_fields", type: :json)
|
138
|
+
assert actual.include?(name: "satisfaction_rating", type: :json)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
sub_test_case "include subresources" do
|
143
|
+
def page_builder
|
144
|
+
@page_builder ||= Class.new do
|
145
|
+
def add(_); end
|
146
|
+
def finish; end
|
147
|
+
end.new
|
148
|
+
end
|
149
|
+
|
150
|
+
sub_test_case "guess" do
|
151
|
+
def task
|
152
|
+
t = {
|
153
|
+
type: "zendesk",
|
154
|
+
login_url: "https://example.zendesk.com/",
|
155
|
+
auth_method: "token",
|
156
|
+
username: "foo@example.com",
|
157
|
+
token: "token",
|
158
|
+
target: "tickets",
|
159
|
+
includes: includes,
|
160
|
+
}
|
161
|
+
t.delete :includes unless includes
|
162
|
+
t
|
163
|
+
end
|
164
|
+
|
165
|
+
def config
|
166
|
+
Embulk::DataSource.new(task)
|
167
|
+
end
|
168
|
+
|
169
|
+
def ticket
|
170
|
+
JSON.parse(fixture_load("tickets.json"))
|
171
|
+
end
|
172
|
+
|
173
|
+
setup do
|
174
|
+
@client = Client.new(task)
|
175
|
+
stub(Client).new { @client }
|
176
|
+
stub(@client).public_send(anything) do |*args|
|
177
|
+
args.last.call(ticket)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
sub_test_case "includes present" do
|
182
|
+
def includes
|
183
|
+
%w(audits comments)
|
184
|
+
end
|
185
|
+
|
186
|
+
test "guessed includes fields" do
|
187
|
+
actual = Plugin.guess(config)["columns"]
|
188
|
+
assert actual.include?(name: "audits", type: :json)
|
189
|
+
assert actual.include?(name: "comments", type: :json)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
sub_test_case "includes blank" do
|
194
|
+
def includes
|
195
|
+
nil
|
196
|
+
end
|
197
|
+
|
198
|
+
test "not guessed includes fields" do
|
199
|
+
actual = Plugin.guess(config)["columns"]
|
200
|
+
assert !actual.include?(name: "audits", type: :json)
|
201
|
+
assert !actual.include?(name: "comments", type: :json)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
sub_test_case "#run" do
|
207
|
+
def schema
|
208
|
+
[
|
209
|
+
{"name" => "id", "type" => "long"},
|
210
|
+
{"name" => "tags", "type" => "json"},
|
211
|
+
]
|
212
|
+
end
|
213
|
+
|
214
|
+
def run_task
|
215
|
+
task.merge({
|
216
|
+
schema: schema,
|
217
|
+
retry_limit: 1,
|
218
|
+
retry_initial_wait_sec: 0,
|
219
|
+
includes: includes,
|
220
|
+
})
|
221
|
+
end
|
222
|
+
|
223
|
+
setup do
|
224
|
+
@client = Client.new(run_task)
|
225
|
+
stub(@client).public_send {|*args| args.last.call({}) }
|
226
|
+
@plugin = Plugin.new(run_task, nil, nil, page_builder)
|
227
|
+
stub(@plugin).client { @client }
|
228
|
+
@httpclient = @client.httpclient
|
229
|
+
stub(@client).httpclient { @httpclient }
|
230
|
+
end
|
231
|
+
|
232
|
+
sub_test_case "preview" do
|
233
|
+
setup do
|
234
|
+
stub(@plugin).preview? { true }
|
235
|
+
end
|
236
|
+
|
237
|
+
sub_test_case "includes present" do
|
238
|
+
def includes
|
239
|
+
%w(foo bar)
|
240
|
+
end
|
241
|
+
|
242
|
+
test "call fetch_subresource" do
|
243
|
+
includes.each do |ent|
|
244
|
+
mock(@client).fetch_subresource(anything, anything, ent).never
|
245
|
+
end
|
246
|
+
@plugin.run
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
sub_test_case "includes blank" do
|
251
|
+
def includes
|
252
|
+
[]
|
253
|
+
end
|
254
|
+
|
255
|
+
test "don't call fetch_subresource" do
|
256
|
+
mock(@client).fetch_subresource.never
|
257
|
+
@plugin.run
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
135
261
|
|
136
|
-
|
137
|
-
|
138
|
-
|
262
|
+
sub_test_case "run" do
|
263
|
+
setup do
|
264
|
+
stub(@plugin).preview? { false }
|
265
|
+
end
|
266
|
+
|
267
|
+
sub_test_case "includes present " do
|
268
|
+
def includes
|
269
|
+
%w(foo bar)
|
270
|
+
end
|
271
|
+
|
272
|
+
test "call fetch_subresource" do
|
273
|
+
includes.each do |ent|
|
274
|
+
mock(@client).fetch_subresource(anything, anything, ent).at_least(1)
|
275
|
+
end
|
276
|
+
@plugin.run
|
277
|
+
end
|
278
|
+
end
|
139
279
|
|
140
|
-
|
141
|
-
|
280
|
+
sub_test_case "includes blank" do
|
281
|
+
def includes
|
282
|
+
[]
|
283
|
+
end
|
284
|
+
|
285
|
+
test "don't call fetch_subresource" do
|
286
|
+
mock(@client).fetch_subresource.never
|
287
|
+
@plugin.run
|
288
|
+
end
|
289
|
+
end
|
290
|
+
end
|
142
291
|
end
|
143
292
|
end
|
144
293
|
|
@@ -149,7 +298,8 @@ module Embulk
|
|
149
298
|
|
150
299
|
def schema
|
151
300
|
[
|
152
|
-
{"name" => "id", "type" => "long"}
|
301
|
+
{"name" => "id", "type" => "long"},
|
302
|
+
{"name" => "tags", "type" => "json"},
|
153
303
|
]
|
154
304
|
end
|
155
305
|
|
@@ -158,6 +308,7 @@ module Embulk
|
|
158
308
|
schema: schema,
|
159
309
|
retry_limit: 1,
|
160
310
|
retry_initial_wait_sec: 0,
|
311
|
+
includes: [],
|
161
312
|
})
|
162
313
|
end
|
163
314
|
|
@@ -184,8 +335,8 @@ module Embulk
|
|
184
335
|
|
185
336
|
test "task[:schema] columns passed into page_builder.add" do
|
186
337
|
tickets = [
|
187
|
-
{"id" => 1, "created_at" => "2000-01-01T00:00:00+0900"},
|
188
|
-
{"id" => 2, "created_at" => "2000-01-01T00:00:00+0900"},
|
338
|
+
{"id" => 1, "created_at" => "2000-01-01T00:00:00+0900", "tags" => ["foo"]},
|
339
|
+
{"id" => 2, "created_at" => "2000-01-01T00:00:00+0900", "tags" => ["foo"]},
|
189
340
|
]
|
190
341
|
|
191
342
|
@httpclient.test_loopback_http_response << [
|
@@ -198,7 +349,7 @@ module Embulk
|
|
198
349
|
].join("\r\n")
|
199
350
|
|
200
351
|
tickets.each do |ticket|
|
201
|
-
mock(page_builder).add([ticket["id"]])
|
352
|
+
mock(page_builder).add([ticket["id"], ticket["tags"]])
|
202
353
|
end
|
203
354
|
mock(page_builder).finish
|
204
355
|
|
@@ -317,6 +468,7 @@ module Embulk
|
|
317
468
|
{"name" => "target_str", "type" => "string"},
|
318
469
|
{"name" => "target_bool", "type" => "boolean"},
|
319
470
|
{"name" => "target_time", "type" => "timestamp"},
|
471
|
+
{"name" => "target_json", "type" => "json"},
|
320
472
|
]
|
321
473
|
end
|
322
474
|
|
@@ -325,22 +477,25 @@ module Embulk
|
|
325
477
|
{
|
326
478
|
"id" => 1, "target_l" => "3", "target_f" => "3", "target_str" => "str",
|
327
479
|
"target_bool" => false, "target_time" => "2000-01-01",
|
480
|
+
"target_json" => [1,2,3],
|
328
481
|
},
|
329
482
|
{
|
330
483
|
"id" => 2, "target_l" => 4.5, "target_f" => 4.5, "target_str" => 999,
|
331
484
|
"target_bool" => "truthy", "target_time" => Time.parse("1999-01-01"),
|
485
|
+
"target_json" => {"foo" => "bar"},
|
332
486
|
},
|
333
487
|
{
|
334
488
|
"id" => 3, "target_l" => nil, "target_f" => nil, "target_str" => nil,
|
335
489
|
"target_bool" => nil, "target_time" => nil,
|
490
|
+
"target_json" => nil,
|
336
491
|
},
|
337
492
|
]
|
338
493
|
end
|
339
494
|
|
340
495
|
test "cast as given type" do
|
341
|
-
mock(page_builder).add([3, 3.0, "str", false, Time.parse("2000-01-01")])
|
342
|
-
mock(page_builder).add([4, 4.5, "999", true, Time.parse("1999-01-01")])
|
343
|
-
mock(page_builder).add([nil, nil, nil, nil, nil])
|
496
|
+
mock(page_builder).add([3, 3.0, "str", false, Time.parse("2000-01-01"), [1,2,3]])
|
497
|
+
mock(page_builder).add([4, 4.5, "999", true, Time.parse("1999-01-01"), {"foo" => "bar"}])
|
498
|
+
mock(page_builder).add([nil, nil, nil, nil, nil, nil])
|
344
499
|
mock(page_builder).finish
|
345
500
|
|
346
501
|
@plugin.run
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-zendesk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2016-
|
13
|
+
date: 2016-03-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|