embulk-input-zendesk 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.ruby-version +1 -0
- data/.travis.yml +44 -0
- data/.travis.yml.erb +43 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile +2 -0
- data/LICENSE.txt +21 -0
- data/README.md +51 -0
- data/Rakefile +21 -0
- data/embulk-input-zendesk.gemspec +28 -0
- data/gemfiles/embulk-0.8.0-latest +4 -0
- data/gemfiles/embulk-0.8.1 +4 -0
- data/gemfiles/embulk-latest +4 -0
- data/gemfiles/template.erb +4 -0
- data/lib/embulk/input/zendesk.rb +9 -0
- data/lib/embulk/input/zendesk/client.rb +199 -0
- data/lib/embulk/input/zendesk/plugin.rb +138 -0
- data/test/capture_io.rb +45 -0
- data/test/embulk/input/zendesk/test_client.rb +469 -0
- data/test/embulk/input/zendesk/test_plugin.rb +338 -0
- data/test/fixture_helper.rb +11 -0
- data/test/fixtures/invalid_lack_username.yml +9 -0
- data/test/fixtures/invalid_unknown_auth.yml +9 -0
- data/test/fixtures/tickets.json +44 -0
- data/test/fixtures/valid_auth_basic.yml +11 -0
- data/test/fixtures/valid_auth_oauth.yml +10 -0
- data/test/fixtures/valid_auth_token.yml +11 -0
- data/test/override_assert_raise.rb +21 -0
- data/test/run-test.rb +26 -0
- metadata +253 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 937ca47ed79588673fdf772d2a55dfe53ee0c433
|
4
|
+
data.tar.gz: 478a4b349a30dbdb019db9c03b21938755ed78c9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a084ce0b8fa7bc6fe13ebf8d4ebeb0535d779928370aab1f54cf3adf3504972cf0f262d48f459da34526c44f3815db710732b368b8e8477da1fe10dfedd3870d
|
7
|
+
data.tar.gz: f404d3e55297b6885b03bb1599fe1b4bb4e5bf50cfdf0a7020d65b0bc2126a3deed9a9939f9aa9d45fe686e50021349db6503bdaecf59a5d82a7683778980f42
|
data/.gitignore
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
jruby-9.0.4.0
|
data/.travis.yml
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
language: ruby
|
2
|
+
|
3
|
+
jdk: oraclejdk8
|
4
|
+
|
5
|
+
before_install:
|
6
|
+
- rvm list known
|
7
|
+
- rvm list
|
8
|
+
- ruby -v
|
9
|
+
- |
|
10
|
+
# Currently, Travis can't treat jruby 9.0.4.0
|
11
|
+
if [[ ${TRAVIS_RUBY_VERSION} != "jruby-9.0.4.0" ]];then
|
12
|
+
rvm get head
|
13
|
+
rvm use jruby-9.0.4.0 --install
|
14
|
+
ruby -v
|
15
|
+
fi
|
16
|
+
- gem install bundler -v 1.10.6
|
17
|
+
|
18
|
+
script: bundle exec rake cov
|
19
|
+
|
20
|
+
rvm:
|
21
|
+
- jruby-9.0.4.0
|
22
|
+
env:
|
23
|
+
# Set on Travis
|
24
|
+
# CODECLIMATE_REPO_TOKEN: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
25
|
+
|
26
|
+
gemfile:
|
27
|
+
- gemfiles/embulk-0.8.0-latest
|
28
|
+
- gemfiles/embulk-0.8.1
|
29
|
+
- gemfiles/embulk-latest
|
30
|
+
|
31
|
+
matrix:
|
32
|
+
exclude:
|
33
|
+
- jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
|
34
|
+
include:
|
35
|
+
- {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-0.8.0-latest}
|
36
|
+
- {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-0.8.1}
|
37
|
+
- {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-latest}
|
38
|
+
|
39
|
+
|
40
|
+
allow_failures:
|
41
|
+
# Ignore failure for *-latest
|
42
|
+
- gemfile: gemfiles/embulk-0.8.0-latest
|
43
|
+
- gemfile: gemfiles/embulk-latest
|
44
|
+
|
data/.travis.yml.erb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
language: ruby
|
2
|
+
|
3
|
+
jdk: oraclejdk8
|
4
|
+
|
5
|
+
before_install:
|
6
|
+
- rvm list known
|
7
|
+
- rvm list
|
8
|
+
- ruby -v
|
9
|
+
- |
|
10
|
+
# Currently, Travis can't treat jruby 9.0.4.0
|
11
|
+
if [[ ${TRAVIS_RUBY_VERSION} != "jruby-9.0.4.0" ]];then
|
12
|
+
rvm get head
|
13
|
+
rvm use jruby-9.0.4.0 --install
|
14
|
+
ruby -v
|
15
|
+
fi
|
16
|
+
- gem install bundler -v 1.10.6
|
17
|
+
|
18
|
+
script: bundle exec rake cov
|
19
|
+
|
20
|
+
rvm:
|
21
|
+
- jruby-9.0.4.0
|
22
|
+
env:
|
23
|
+
# Set on Travis
|
24
|
+
# CODECLIMATE_REPO_TOKEN: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
25
|
+
|
26
|
+
gemfile:
|
27
|
+
<% versions.each do |file| -%>
|
28
|
+
- gemfiles/<%= file %>
|
29
|
+
<% end -%>
|
30
|
+
|
31
|
+
matrix:
|
32
|
+
exclude:
|
33
|
+
- jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
|
34
|
+
include:
|
35
|
+
<% matrix.each do |m| -%>
|
36
|
+
<%= m.gsub("9.0.0.0", "9.0.4.0") %>
|
37
|
+
<% end %>
|
38
|
+
|
39
|
+
allow_failures:
|
40
|
+
# Ignore failure for *-latest
|
41
|
+
<% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
|
42
|
+
- gemfile: gemfiles/<%= file %>
|
43
|
+
<% end %>
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
|
2
|
+
MIT License
|
3
|
+
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
a copy of this software and associated documentation files (the
|
6
|
+
"Software"), to deal in the Software without restriction, including
|
7
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/treasure-data/embulk-input-zendesk.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-zendesk)
|
2
|
+
[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-zendesk)
|
3
|
+
[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/coverage)
|
4
|
+
[![Gem Version](https://badge.fury.io/rb/embulk-input-zendesk.svg)](https://badge.fury.io/rb/embulk-input-zendesk)
|
5
|
+
|
6
|
+
# Zendesk input plugin for Embulk
|
7
|
+
|
8
|
+
Embulk input plugin for loading [Zendesk](https://www.zendesk.com/) records.
|
9
|
+
|
10
|
+
## Overview
|
11
|
+
|
12
|
+
Required Embulk version >= 0.8.1.
|
13
|
+
|
14
|
+
**NOTE** This plugin don't support JSON type columns e.g. custom fields, tags, etc for now. But they will be supported soon.
|
15
|
+
|
16
|
+
* **Plugin type**: input
|
17
|
+
* **Resume supported**: no
|
18
|
+
* **Cleanup supported**: no
|
19
|
+
* **Guess supported**: yes
|
20
|
+
|
21
|
+
## Configuration
|
22
|
+
|
23
|
+
- **login_url**: Login URL for Zendesk (string, required)
|
24
|
+
- **auth_method**: `basic`, `token`, or `oauth`. For more detail on [zendesk document](https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication). (string, required)
|
25
|
+
- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields` or `ticket_forms`. (string, required)
|
26
|
+
- **username**: The user name a.k.a. email. Required if `auth_method` is `basic` or `token`. (string, default: `null`)
|
27
|
+
- **password**: Password. required if `auth_method` is `basic`. (string, default: `null`)
|
28
|
+
- **token**: Token. required if `auth_method` is `token`. (string, default: `null`)
|
29
|
+
- **access_token**: OAuth Access Token. required if `auth_method` is `oauth`. (string, default: `null`)
|
30
|
+
- **start_time**: Start export from this time if present. (string, default: `null`)
|
31
|
+
- **retry_limit**: Try to retry this times (integer, default: 5)
|
32
|
+
- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1)
|
33
|
+
|
34
|
+
## Example
|
35
|
+
|
36
|
+
```yaml
|
37
|
+
in:
|
38
|
+
type: zendesk
|
39
|
+
login_url: https://obscura.zendesk.com
|
40
|
+
auth_method: token
|
41
|
+
username: jdoe@example.com
|
42
|
+
token: 6wiIBWbGkBMo1mRDMuVwkw1EPsNkeUj95PIz2akv
|
43
|
+
start_time: "2015-01-01 00:00:00+0000"
|
44
|
+
```
|
45
|
+
|
46
|
+
|
47
|
+
## Test
|
48
|
+
|
49
|
+
```
|
50
|
+
$ rake test
|
51
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "everyleaf/embulk_helper/tasks"
|
3
|
+
|
4
|
+
task default: :test
|
5
|
+
|
6
|
+
desc "Run tests"
|
7
|
+
task :test do
|
8
|
+
ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
|
9
|
+
end
|
10
|
+
|
11
|
+
desc "Run tests with coverage"
|
12
|
+
task :cov do
|
13
|
+
ENV["COVERAGE"] = "1"
|
14
|
+
ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
Everyleaf::EmbulkHelper::Tasks.install(
|
19
|
+
gemspec: "./embulk-input-zendesk.gemspec",
|
20
|
+
github_name: "treasure-data/embulk-input-zendesk",
|
21
|
+
)
|
@@ -0,0 +1,28 @@
|
|
1
|
+
|
2
|
+
Gem::Specification.new do |spec|
|
3
|
+
spec.name = "embulk-input-zendesk"
|
4
|
+
spec.version = "0.1.0"
|
5
|
+
spec.authors = ["uu59"]
|
6
|
+
spec.summary = "Zendesk input plugin for Embulk"
|
7
|
+
spec.description = "Loads records from Zendesk."
|
8
|
+
spec.email = ["k@uu59.org"]
|
9
|
+
spec.licenses = ["MIT"]
|
10
|
+
# TODO set this: spec.homepage = "https://github.com/k/embulk-input-zendesk"
|
11
|
+
|
12
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
14
|
+
spec.require_paths = ["lib"]
|
15
|
+
|
16
|
+
spec.add_dependency 'perfect_retry', '~> 0.4'
|
17
|
+
spec.add_dependency 'httpclient'
|
18
|
+
spec.add_development_dependency 'embulk', ['~> 0.8.1']
|
19
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
20
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
21
|
+
spec.add_development_dependency 'pry'
|
22
|
+
spec.add_development_dependency 'test-unit', '~> 3.1.5'
|
23
|
+
spec.add_development_dependency 'test-unit-rr'
|
24
|
+
spec.add_development_dependency 'rr', '~> 1.1.2'
|
25
|
+
spec.add_development_dependency 'simplecov'
|
26
|
+
spec.add_development_dependency 'everyleaf-embulk_helper'
|
27
|
+
spec.add_development_dependency "codeclimate-test-reporter"
|
28
|
+
end
|
@@ -0,0 +1,199 @@
|
|
1
|
+
require "httpclient"
|
2
|
+
|
3
|
+
module Embulk
|
4
|
+
module Input
|
5
|
+
module Zendesk
|
6
|
+
class Client
|
7
|
+
attr_reader :config
|
8
|
+
|
9
|
+
PARTIAL_RECORDS_SIZE = 50
|
10
|
+
AVAILABLE_TARGETS = %w(
|
11
|
+
tickets ticket_events users organizations
|
12
|
+
ticket_fields ticket_forms
|
13
|
+
).freeze
|
14
|
+
|
15
|
+
def initialize(config)
|
16
|
+
@config = config
|
17
|
+
end
|
18
|
+
|
19
|
+
def httpclient
|
20
|
+
httpclient = HTTPClient.new
|
21
|
+
# httpclient.debug_dev = STDOUT
|
22
|
+
return set_auth(httpclient)
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate_config
|
26
|
+
validate_credentials
|
27
|
+
validate_target
|
28
|
+
end
|
29
|
+
|
30
|
+
def validate_credentials
|
31
|
+
valid = case config[:auth_method]
|
32
|
+
when "basic"
|
33
|
+
config[:username] && config[:password]
|
34
|
+
when "token"
|
35
|
+
config[:username] && config[:token]
|
36
|
+
when "oauth"
|
37
|
+
config[:access_token]
|
38
|
+
else
|
39
|
+
raise Embulk::ConfigError.new("Unknown auth_method (#{config[:auth_method]}). Should pick one from 'basic', 'token' or 'oauth'.")
|
40
|
+
end
|
41
|
+
|
42
|
+
unless valid
|
43
|
+
raise Embulk::ConfigError.new("Missing required credentials for #{config[:auth_method]}")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def validate_target
|
48
|
+
unless AVAILABLE_TARGETS.include?(config[:target])
|
49
|
+
raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported.")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
%w(tickets users organizations).each do |target|
|
54
|
+
define_method(target) do |partial = true, start_time = 0, &block|
|
55
|
+
if partial
|
56
|
+
export("/api/v2/#{target}.json", target, PARTIAL_RECORDS_SIZE, &block) # Ignore start_time
|
57
|
+
else
|
58
|
+
incremental_export("/api/v2/incremental/#{target}.json", target, start_time, [], &block)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def ticket_events(partial = true, start_time = 0, &block)
|
64
|
+
# NOTE: ticket_events only have incremental export API
|
65
|
+
path = "/api/v2/incremental/ticket_events"
|
66
|
+
incremental_export(path, "ticket_events", start_time, [], &block)
|
67
|
+
end
|
68
|
+
|
69
|
+
def ticket_fields(partial = true, start_time = 0, &block)
|
70
|
+
# NOTE: ticket_fields only have export API (not incremental)
|
71
|
+
path = "/api/v2/ticket_fields.json"
|
72
|
+
export(path, "ticket_fields", 1000, &block)
|
73
|
+
end
|
74
|
+
|
75
|
+
def ticket_forms(partial = true, start_time = 0, &block)
|
76
|
+
# NOTE: ticket_forms only have export API (not incremental)
|
77
|
+
path = "/api/v2/ticket_forms.json"
|
78
|
+
export(path, "ticket_forms", 1000, &block)
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def export(path, key, per_page, &block)
|
84
|
+
# for `embulk guess` and `embulk preview` to fetch ~50 records only.
|
85
|
+
# incremental export API has supported only 1000 per page, it is too large to guess/preview
|
86
|
+
Embulk.logger.debug "#{path} with per_page: #{per_page}"
|
87
|
+
response = request(path, per_page: per_page)
|
88
|
+
|
89
|
+
begin
|
90
|
+
data = JSON.parse(response.body)
|
91
|
+
rescue => e
|
92
|
+
raise Embulk::DataError.new(e)
|
93
|
+
end
|
94
|
+
|
95
|
+
data[key].each do |record|
|
96
|
+
block.call record
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def incremental_export(path, key, start_time = 0, known_ids = [], &block)
|
101
|
+
# for `embulk run` to fetch all records.
|
102
|
+
response = request(path, start_time: start_time)
|
103
|
+
|
104
|
+
begin
|
105
|
+
data = JSON.parse(response.body)
|
106
|
+
rescue => e
|
107
|
+
raise Embulk::DataError.new(e)
|
108
|
+
end
|
109
|
+
|
110
|
+
Embulk.logger.debug "start_time:#{start_time} (#{Time.at(start_time)}) count:#{data["count"]} next_page:#{data["next_page"]} end_time:#{data["end_time"]} "
|
111
|
+
data[key].each do |record|
|
112
|
+
# de-duplicated records.
|
113
|
+
# https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
|
114
|
+
# https://github.com/zendesk/zendesk_api_client_rb/issues/251
|
115
|
+
next if known_ids.include?(record["id"])
|
116
|
+
|
117
|
+
known_ids << record["id"]
|
118
|
+
block.call record
|
119
|
+
end
|
120
|
+
|
121
|
+
# NOTE: If count is less than 1000, then stop paginating.
|
122
|
+
# Otherwise, use the next_page URL to get the next page of results.
|
123
|
+
# https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
|
124
|
+
if data["count"] == 1000
|
125
|
+
incremental_export(path, key, data["end_time"], known_ids, &block)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def retryer
|
130
|
+
PerfectRetry.new do |config|
|
131
|
+
config.limit = @config[:retry_limit]
|
132
|
+
config.logger = Embulk.logger
|
133
|
+
config.log_level = nil
|
134
|
+
config.dont_rescues = [Embulk::DataError, Embulk::ConfigError]
|
135
|
+
config.sleep = lambda{|n| @config[:retry_initial_wait_sec]* (2 ** (n-1)) }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def set_auth(httpclient)
|
140
|
+
validate_credentials
|
141
|
+
|
142
|
+
# https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication
|
143
|
+
case config[:auth_method]
|
144
|
+
when "basic"
|
145
|
+
httpclient.set_auth(config[:login_url], config[:username], config[:password])
|
146
|
+
when "token"
|
147
|
+
httpclient.set_auth(config[:login_url], "#{config[:username]}/token", config[:token])
|
148
|
+
when "oauth"
|
149
|
+
httpclient.default_header = {
|
150
|
+
"Authorization" => "Bearer #{config[:access_token]}"
|
151
|
+
}
|
152
|
+
end
|
153
|
+
httpclient
|
154
|
+
end
|
155
|
+
|
156
|
+
def request(path, query = {})
|
157
|
+
u = URI.parse(config[:login_url])
|
158
|
+
u.path = path
|
159
|
+
|
160
|
+
retryer.with_retry do
|
161
|
+
response = httpclient.get(u.to_s, query, follow_redirect: true)
|
162
|
+
|
163
|
+
# https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
|
164
|
+
status_code = response.status
|
165
|
+
case status_code
|
166
|
+
when 200
|
167
|
+
response
|
168
|
+
when 400, 401
|
169
|
+
raise Embulk::ConfigError.new("[#{status_code}] #{response.body}")
|
170
|
+
when 409
|
171
|
+
raise "[#{status_code}] temporally failure."
|
172
|
+
when 429
|
173
|
+
# rate limit
|
174
|
+
retry_after = response.headers["Retry-After"]
|
175
|
+
wait_rate_limit(retry_after.to_i)
|
176
|
+
when 500, 503
|
177
|
+
# 503 is possible rate limit
|
178
|
+
retry_after = response.headers["Retry-After"]
|
179
|
+
if retry_after
|
180
|
+
wait_rate_limit(retry_after.to_i)
|
181
|
+
else
|
182
|
+
raise "[#{status_code}] temporally failure."
|
183
|
+
end
|
184
|
+
else
|
185
|
+
raise "Server returns unknown status code (#{status_code})"
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def wait_rate_limit(retry_after)
|
191
|
+
Embulk.logger.warn "Rate Limited. Waiting #{retry_after} seconds to retry"
|
192
|
+
sleep retry_after
|
193
|
+
throw :retry
|
194
|
+
end
|
195
|
+
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|