embulk-input-zendesk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 937ca47ed79588673fdf772d2a55dfe53ee0c433
+  data.tar.gz: 478a4b349a30dbdb019db9c03b21938755ed78c9
+SHA512:
+  metadata.gz: a084ce0b8fa7bc6fe13ebf8d4ebeb0535d779928370aab1f54cf3adf3504972cf0f262d48f459da34526c44f3815db710732b368b8e8477da1fe10dfedd3870d
+  data.tar.gz: f404d3e55297b6885b03bb1599fe1b4bb4e5bf50cfdf0a7020d65b0bc2126a3deed9a9939f9aa9d45fe686e50021349db6503bdaecf59a5d82a7683778980f42

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock
+/coverage/

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.4.0

data/.travis.yml

@@ -0,0 +1,44 @@
+language: ruby
+
+jdk: oraclejdk8
+
+before_install:
+  - rvm list known
+  - rvm list
+  - ruby -v
+  - |
+    # Currently, Travis can't treat jruby 9.0.4.0
+    if [[ ${TRAVIS_RUBY_VERSION} != "jruby-9.0.4.0" ]];then
+      rvm get head
+      rvm use jruby-9.0.4.0 --install
+      ruby -v
+    fi
+  - gem install bundler -v 1.10.6
+
+script: bundle exec rake cov
+
+rvm:
+  - jruby-9.0.4.0
+env:
+  # Set on Travis
+  # CODECLIMATE_REPO_TOKEN: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+gemfile:
+  - gemfiles/embulk-0.8.0-latest
+  - gemfiles/embulk-0.8.1
+  - gemfiles/embulk-latest
+
+matrix:
+  exclude:
+    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
+  include:
+    - {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-0.8.0-latest}
+    - {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-0.8.1}
+    - {rvm: jruby-9.0.4.0, gemfile: gemfiles/embulk-latest}
+    
+
+  allow_failures:
+    # Ignore failure for *-latest
+    - gemfile: gemfiles/embulk-0.8.0-latest
+    - gemfile: gemfiles/embulk-latest
+    

data/.travis.yml.erb

@@ -0,0 +1,43 @@
+language: ruby
+
+jdk: oraclejdk8
+
+before_install:
+  - rvm list known
+  - rvm list
+  - ruby -v
+  - |
+    # Currently, Travis can't treat jruby 9.0.4.0
+    if [[ ${TRAVIS_RUBY_VERSION} != "jruby-9.0.4.0" ]];then
+      rvm get head
+      rvm use jruby-9.0.4.0 --install
+      ruby -v
+    fi
+  - gem install bundler -v 1.10.6
+
+script: bundle exec rake cov
+
+rvm:
+  - jruby-9.0.4.0
+env:
+  # Set on Travis
+  # CODECLIMATE_REPO_TOKEN: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+gemfile:
+<% versions.each do |file| -%>
+  - gemfiles/<%= file %>
+<% end -%>
+
+matrix:
+  exclude:
+    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
+  include:
+    <% matrix.each do |m| -%>
+<%= m.gsub("9.0.0.0", "9.0.4.0") %>
+    <% end %>
+
+  allow_failures:
+    # Ignore failure for *-latest
+    <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
+- gemfile: gemfiles/<%= file %>
+    <% end %>

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 - 2016-01-26
+
+The first release!!

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+[![Build Status](https://travis-ci.org/treasure-data/embulk-input-zendesk.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-zendesk)
+[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-zendesk)
+[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-zendesk/coverage)
+[![Gem Version](https://badge.fury.io/rb/embulk-input-zendesk.svg)](https://badge.fury.io/rb/embulk-input-zendesk)
+
+# Zendesk input plugin for Embulk
+
+Embulk input plugin for loading [Zendesk](https://www.zendesk.com/) records.
+
+## Overview
+
+Required Embulk version >= 0.8.1.
+
+**NOTE** This plugin don't support JSON type columns e.g. custom fields, tags, etc for now. But they will be supported soon.
+
+* **Plugin type**: input
+* **Resume supported**: no
+* **Cleanup supported**: no
+* **Guess supported**: yes
+
+## Configuration
+
+- **login_url**: Login URL for Zendesk (string, required)
+- **auth_method**: `basic`, `token`, or `oauth`. For more detail on [zendesk document](https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication). (string, required)
+- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields` or `ticket_forms`. (string, required)
+- **username**: The user name a.k.a. email. Required if `auth_method` is `basic` or `token`. (string, default: `null`)
+- **password**: Password. required if `auth_method` is `basic`. (string, default: `null`)
+- **token**: Token. required if `auth_method` is `token`. (string, default: `null`)
+- **access_token**: OAuth Access Token. required if `auth_method` is `oauth`. (string, default: `null`)
+- **start_time**: Start export from this time if present. (string, default: `null`)
+- **retry_limit**: Try to retry this times (integer, default: 5)
+- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1)
+
+## Example
+
+```yaml
+in:
+  type: zendesk
+  login_url: https://obscura.zendesk.com
+  auth_method: token
+  username: jdoe@example.com
+  token: 6wiIBWbGkBMo1mRDMuVwkw1EPsNkeUj95PIz2akv
+  start_time: "2015-01-01 00:00:00+0000"
+```
+
+
+## Test
+
+```
+$ rake test
+```

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/gem_tasks"
+require "everyleaf/embulk_helper/tasks"
+
+task default: :test
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+desc "Run tests with coverage"
+task :cov do
+  ENV["COVERAGE"] = "1"
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+
+Everyleaf::EmbulkHelper::Tasks.install(
+  gemspec: "./embulk-input-zendesk.gemspec",
+  github_name: "treasure-data/embulk-input-zendesk",
+)

data/embulk-input-zendesk.gemspec

@@ -0,0 +1,28 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-zendesk"
+  spec.version       = "0.1.0"
+  spec.authors       = ["uu59"]
+  spec.summary       = "Zendesk input plugin for Embulk"
+  spec.description   = "Loads records from Zendesk."
+  spec.email         = ["k@uu59.org"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/k/embulk-input-zendesk"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'perfect_retry', '~> 0.4'
+  spec.add_dependency 'httpclient'
+  spec.add_development_dependency 'embulk', ['~> 0.8.1']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'test-unit', '~> 3.1.5'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'rr', '~> 1.1.2'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'everyleaf-embulk_helper'
+  spec.add_development_dependency "codeclimate-test-reporter"
+end

data/gemfiles/embulk-0.8.0-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "~> 0.8.0"

data/gemfiles/embulk-0.8.1

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.8.1"

data/gemfiles/embulk-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "> 0.8.1"

data/gemfiles/template.erb

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "<%= version %>"

data/lib/embulk/input/zendesk.rb

@@ -0,0 +1,9 @@
+require "embulk/input/zendesk/client"
+require "embulk/input/zendesk/plugin"
+
+module Embulk
+  module Input
+    module Zendesk
+    end
+  end
+end

data/lib/embulk/input/zendesk/client.rb

@@ -0,0 +1,199 @@
+require "httpclient"
+
+module Embulk
+  module Input
+    module Zendesk
+      class Client
+        attr_reader :config
+
+        PARTIAL_RECORDS_SIZE = 50
+        AVAILABLE_TARGETS = %w(
+          tickets ticket_events users organizations
+          ticket_fields ticket_forms
+        ).freeze
+
+        def initialize(config)
+          @config = config
+        end
+
+        def httpclient
+          httpclient = HTTPClient.new
+          # httpclient.debug_dev = STDOUT
+          return set_auth(httpclient)
+        end
+
+        def validate_config
+          validate_credentials
+          validate_target
+        end
+
+        def validate_credentials
+          valid = case config[:auth_method]
+          when "basic"
+            config[:username] && config[:password]
+          when "token"
+            config[:username] && config[:token]
+          when "oauth"
+            config[:access_token]
+          else
+            raise Embulk::ConfigError.new("Unknown auth_method (#{config[:auth_method]}). Should pick one from 'basic', 'token' or 'oauth'.")
+          end
+
+          unless valid
+            raise Embulk::ConfigError.new("Missing required credentials for #{config[:auth_method]}")
+          end
+        end
+
+        def validate_target
+          unless AVAILABLE_TARGETS.include?(config[:target])
+            raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported.")
+          end
+        end
+
+        %w(tickets users organizations).each do |target|
+          define_method(target) do |partial = true, start_time = 0, &block|
+            if partial
+              export("/api/v2/#{target}.json", target, PARTIAL_RECORDS_SIZE, &block) # Ignore start_time
+            else
+              incremental_export("/api/v2/incremental/#{target}.json", target, start_time, [], &block)
+            end
+          end
+        end
+
+        def ticket_events(partial = true, start_time = 0, &block)
+          # NOTE: ticket_events only have incremental export API
+          path = "/api/v2/incremental/ticket_events"
+          incremental_export(path, "ticket_events", start_time, [], &block)
+        end
+
+        def ticket_fields(partial = true, start_time = 0, &block)
+          # NOTE: ticket_fields only have export API (not incremental)
+          path = "/api/v2/ticket_fields.json"
+          export(path, "ticket_fields", 1000, &block)
+        end
+
+        def ticket_forms(partial = true, start_time = 0, &block)
+          # NOTE: ticket_forms only have export API (not incremental)
+          path = "/api/v2/ticket_forms.json"
+          export(path, "ticket_forms", 1000, &block)
+        end
+
+        private
+
+        def export(path, key, per_page, &block)
+          # for `embulk guess` and `embulk preview` to fetch ~50 records only.
+          # incremental export API has supported only 1000 per page, it is too large to guess/preview
+          Embulk.logger.debug "#{path} with per_page: #{per_page}"
+          response = request(path, per_page: per_page)
+
+          begin
+            data = JSON.parse(response.body)
+          rescue => e
+            raise Embulk::DataError.new(e)
+          end
+
+          data[key].each do |record|
+            block.call record
+          end
+        end
+
+        def incremental_export(path, key, start_time = 0, known_ids = [], &block)
+          # for `embulk run` to fetch all records.
+          response = request(path, start_time: start_time)
+
+          begin
+            data = JSON.parse(response.body)
+          rescue => e
+            raise Embulk::DataError.new(e)
+          end
+
+          Embulk.logger.debug "start_time:#{start_time} (#{Time.at(start_time)}) count:#{data["count"]} next_page:#{data["next_page"]} end_time:#{data["end_time"]} "
+          data[key].each do |record|
+            # de-duplicated records.
+            # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
+            # https://github.com/zendesk/zendesk_api_client_rb/issues/251
+            next if known_ids.include?(record["id"])
+
+            known_ids << record["id"]
+            block.call record
+          end
+
+          # NOTE: If count is less than 1000, then stop paginating.
+          #       Otherwise, use the next_page URL to get the next page of results.
+          #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
+          if data["count"] == 1000
+            incremental_export(path, key, data["end_time"], known_ids, &block)
+          end
+        end
+
+        def retryer
+          PerfectRetry.new do |config|
+            config.limit = @config[:retry_limit]
+            config.logger = Embulk.logger
+            config.log_level = nil
+            config.dont_rescues = [Embulk::DataError, Embulk::ConfigError]
+            config.sleep = lambda{|n| @config[:retry_initial_wait_sec]* (2 ** (n-1)) }
+          end
+        end
+
+        def set_auth(httpclient)
+          validate_credentials
+
+          # https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication
+          case config[:auth_method]
+          when "basic"
+            httpclient.set_auth(config[:login_url], config[:username], config[:password])
+          when "token"
+            httpclient.set_auth(config[:login_url], "#{config[:username]}/token", config[:token])
+          when "oauth"
+            httpclient.default_header = {
+              "Authorization" => "Bearer #{config[:access_token]}"
+            }
+          end
+          httpclient
+        end
+
+        def request(path, query = {})
+          u = URI.parse(config[:login_url])
+          u.path = path
+
+          retryer.with_retry do
+            response = httpclient.get(u.to_s, query, follow_redirect: true)
+
+            # https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
+            status_code = response.status
+            case status_code
+            when 200
+              response
+            when 400, 401
+              raise Embulk::ConfigError.new("[#{status_code}] #{response.body}")
+            when 409
+              raise "[#{status_code}] temporally failure."
+            when 429
+              # rate limit
+              retry_after = response.headers["Retry-After"]
+              wait_rate_limit(retry_after.to_i)
+            when 500, 503
+              # 503 is possible rate limit
+              retry_after = response.headers["Retry-After"]
+              if retry_after
+                wait_rate_limit(retry_after.to_i)
+              else
+                raise "[#{status_code}] temporally failure."
+              end
+            else
+              raise "Server returns unknown status code (#{status_code})"
+            end
+          end
+        end
+
+        def wait_rate_limit(retry_after)
+          Embulk.logger.warn "Rate Limited. Waiting #{retry_after} seconds to retry"
+          sleep retry_after
+          throw :retry
+        end
+
+      end
+    end
+  end
+end