embulk-input-zendesk 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/README.md +2 -1
- data/embulk-input-zendesk.gemspec +4 -4
- data/lib/embulk/input/zendesk/client.rb +5 -0
- data/lib/embulk/input/zendesk/plugin.rb +18 -2
- data/test/embulk/input/zendesk/test_plugin.rb +55 -0
- metadata +7 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eddb031e2e41652c6c825d2caaefaf618125fa07
|
4
|
+
data.tar.gz: 2b76a1a7a750008b20823c8dbb7f4794ad3d92dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb4d2dc9a0b128bf2c66f259750b9db6f27f9d67b512e48a94cd1d4c4dfd1aa52b8676c37a9f5449d43f6118605bb820d3b06e5f7b5b2020af147c172d85881f
|
7
|
+
data.tar.gz: 519e20eebcb4a732174b6b71be9ae01f097b00108b2d47e71299a4adc6b205fbf34a9c160beebdf31acf7609e2ac9103b6b615a2776838e4658aa1fb1bbb41b3
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
|
+
## 0.1.2 - 2016-01-29
|
2
|
+
|
3
|
+
* [maintenance] Add authors @muga and @sakama.
|
4
|
+
* [enhancement] Add Incremental option [#6](https://github.com/treasure-data/embulk-input-zendesk/pull/6)
|
5
|
+
|
1
6
|
## 0.1.1 - 2016-01-26
|
2
7
|
|
3
|
-
|
8
|
+
* [fixed] Fix when null value given.
|
4
9
|
|
5
10
|
## 0.1.0 - 2016-01-26
|
6
11
|
|
data/README.md
CHANGED
@@ -14,7 +14,7 @@ Required Embulk version >= 0.8.1.
|
|
14
14
|
**NOTE** This plugin don't support JSON type columns e.g. custom fields, tags, etc for now. But they will be supported soon.
|
15
15
|
|
16
16
|
* **Plugin type**: input
|
17
|
-
* **Resume supported**:
|
17
|
+
* **Resume supported**: yes
|
18
18
|
* **Cleanup supported**: no
|
19
19
|
* **Guess supported**: yes
|
20
20
|
|
@@ -30,6 +30,7 @@ Required Embulk version >= 0.8.1.
|
|
30
30
|
- **start_time**: Start export from this time if present. (string, default: `null`)
|
31
31
|
- **retry_limit**: Try to retry this times (integer, default: 5)
|
32
32
|
- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1)
|
33
|
+
- **incremental**: If false, `start_time` in next.yml would not be updated that means you always fetch all of data from Zendesk with statically conditions. If true, `start_time` would be updated in next.yml. (bool, default: true)
|
33
34
|
|
34
35
|
## Example
|
35
36
|
|
@@ -1,13 +1,13 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-input-zendesk"
|
4
|
-
spec.version = "0.1.
|
5
|
-
spec.authors = ["uu59"]
|
4
|
+
spec.version = "0.1.2"
|
5
|
+
spec.authors = ["uu59", "muga", "sakama"]
|
6
6
|
spec.summary = "Zendesk input plugin for Embulk"
|
7
7
|
spec.description = "Loads records from Zendesk."
|
8
|
-
spec.email = ["k@uu59.org"]
|
8
|
+
spec.email = ["k@uu59.org", "muga.nishizawa@gmail.com", "satoshiakama@gmail.com"]
|
9
9
|
spec.licenses = ["MIT"]
|
10
|
-
|
10
|
+
spec.homepage = "https://github.com/treasure-data/embulk-input-zendesk"
|
11
11
|
|
12
12
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
@@ -11,6 +11,7 @@ module Embulk
|
|
11
11
|
tickets ticket_events users organizations
|
12
12
|
ticket_fields ticket_forms
|
13
13
|
).freeze
|
14
|
+
AVAILABLE_INCREMENTAL_EXPORT = AVAILABLE_TARGETS - %w(ticket_fields ticket_forms)
|
14
15
|
|
15
16
|
def initialize(config)
|
16
17
|
@config = config
|
@@ -95,6 +96,8 @@ module Embulk
|
|
95
96
|
data[key].each do |record|
|
96
97
|
block.call record
|
97
98
|
end
|
99
|
+
|
100
|
+
nil # this is necessary different with incremental_export
|
98
101
|
end
|
99
102
|
|
100
103
|
def incremental_export(path, key, start_time = 0, known_ids = [], &block)
|
@@ -123,6 +126,8 @@ module Embulk
|
|
123
126
|
# https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
|
124
127
|
if data["count"] == 1000
|
125
128
|
incremental_export(path, key, data["end_time"], known_ids, &block)
|
129
|
+
else
|
130
|
+
data
|
126
131
|
end
|
127
132
|
end
|
128
133
|
|
@@ -18,13 +18,21 @@ module Embulk
|
|
18
18
|
Column.new(nil, name, type, column["format"])
|
19
19
|
end
|
20
20
|
|
21
|
+
if task[:incremental] && !Client::AVAILABLE_INCREMENTAL_EXPORT.include?(task[:target])
|
22
|
+
Embulk.logger.warn "target: #{task[:target]} don't support incremental export API. Will be ignored start_time option"
|
23
|
+
end
|
24
|
+
|
21
25
|
resume(task, columns, 1, &control)
|
22
26
|
end
|
23
27
|
|
24
28
|
def self.resume(task, columns, count, &control)
|
25
29
|
task_reports = yield(task, columns, count)
|
30
|
+
report = task_reports.first
|
26
31
|
|
27
32
|
next_config_diff = {}
|
33
|
+
if report[:start_time]
|
34
|
+
next_config_diff[:start_time] = report[:start_time]
|
35
|
+
end
|
28
36
|
return next_config_diff
|
29
37
|
end
|
30
38
|
|
@@ -71,6 +79,7 @@ module Embulk
|
|
71
79
|
start_time: config.param("start_time", :string, default: nil),
|
72
80
|
retry_limit: config.param("retry_limit", :integer, default: 5),
|
73
81
|
retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
|
82
|
+
incremental: config.param("incremental", :bool, default: true),
|
74
83
|
schema: config.param(:columns, :array, default: []),
|
75
84
|
}
|
76
85
|
end
|
@@ -87,14 +96,21 @@ module Embulk
|
|
87
96
|
end
|
88
97
|
|
89
98
|
client = Client.new(task)
|
90
|
-
client.public_send(method, *args) do |record|
|
99
|
+
last_data = client.public_send(method, *args) do |record|
|
91
100
|
values = extract_values(record)
|
92
101
|
page_builder.add(values)
|
93
102
|
end
|
94
|
-
|
95
103
|
page_builder.finish
|
96
104
|
|
97
105
|
task_report = {}
|
106
|
+
if task[:incremental] && last_data && last_data["end_time"]
|
107
|
+
# NOTE: start_time compared as "=>", not ">".
|
108
|
+
# If we will use end_time for next start_time, we got the same record that is last fetched
|
109
|
+
# end_time + 1 is workaround for that
|
110
|
+
next_start_time = Time.at(last_data["end_time"] + 1)
|
111
|
+
task_report[:start_time] = next_start_time.strftime("%Y-%m-%d %H:%M:%S%z")
|
112
|
+
end
|
113
|
+
|
98
114
|
return task_report
|
99
115
|
end
|
100
116
|
|
@@ -242,6 +242,61 @@ module Embulk
|
|
242
242
|
@plugin.run
|
243
243
|
end
|
244
244
|
|
245
|
+
sub_test_case "config diff" do
|
246
|
+
def end_time
|
247
|
+
1234567890
|
248
|
+
end
|
249
|
+
|
250
|
+
def next_start_time
|
251
|
+
Time.at(end_time + 1).strftime("%F %T%z")
|
252
|
+
end
|
253
|
+
|
254
|
+
def start_time
|
255
|
+
Time.at(1111111111).strftime("%F %T%z")
|
256
|
+
end
|
257
|
+
|
258
|
+
setup do
|
259
|
+
events = [
|
260
|
+
{"id" => 1, "created_at" => "2000-01-01T00:00:00+0900"},
|
261
|
+
{"id" => 2, "created_at" => "2000-01-01T01:00:00+0900"},
|
262
|
+
]
|
263
|
+
|
264
|
+
@httpclient.test_loopback_http_response << [
|
265
|
+
"HTTP/1.1 200",
|
266
|
+
"Content-Type: application/json",
|
267
|
+
"",
|
268
|
+
{
|
269
|
+
ticket_events: events,
|
270
|
+
end_time: end_time,
|
271
|
+
}.to_json
|
272
|
+
].join("\r\n")
|
273
|
+
stub(page_builder).add(anything)
|
274
|
+
stub(page_builder).finish
|
275
|
+
end
|
276
|
+
|
277
|
+
sub_test_case "incremental: true" do
|
278
|
+
def run_task
|
279
|
+
task.merge(schema: schema, target: "ticket_events", incremental: true, start_time: start_time)
|
280
|
+
end
|
281
|
+
|
282
|
+
test "task_report contains next start_time" do
|
283
|
+
report = @plugin.run
|
284
|
+
assert_equal next_start_time, report[:start_time]
|
285
|
+
end
|
286
|
+
end
|
287
|
+
|
288
|
+
sub_test_case "incremental: false" do
|
289
|
+
def run_task
|
290
|
+
task.merge(schema: schema, target: "ticket_events", incremental: false, start_time: start_time)
|
291
|
+
end
|
292
|
+
|
293
|
+
test "task_report don't contains start_time" do
|
294
|
+
report = @plugin.run
|
295
|
+
assert_nil report[:start_time]
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
245
300
|
sub_test_case "casting value" do
|
246
301
|
setup do
|
247
302
|
stub(@plugin).preview? { false }
|
metadata
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-zendesk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
8
|
+
- muga
|
9
|
+
- sakama
|
8
10
|
autorequire:
|
9
11
|
bindir: bin
|
10
12
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
13
|
+
date: 2016-01-29 00:00:00.000000000 Z
|
12
14
|
dependencies:
|
13
15
|
- !ruby/object:Gem::Dependency
|
14
16
|
requirement: !ruby/object:Gem::Requirement
|
@@ -181,6 +183,8 @@ dependencies:
|
|
181
183
|
description: Loads records from Zendesk.
|
182
184
|
email:
|
183
185
|
- k@uu59.org
|
186
|
+
- muga.nishizawa@gmail.com
|
187
|
+
- satoshiakama@gmail.com
|
184
188
|
executables: []
|
185
189
|
extensions: []
|
186
190
|
extra_rdoc_files: []
|
@@ -214,7 +218,7 @@ files:
|
|
214
218
|
- test/fixtures/valid_auth_token.yml
|
215
219
|
- test/override_assert_raise.rb
|
216
220
|
- test/run-test.rb
|
217
|
-
homepage:
|
221
|
+
homepage: https://github.com/treasure-data/embulk-input-zendesk
|
218
222
|
licenses:
|
219
223
|
- MIT
|
220
224
|
metadata: {}
|