embulk-input-zendesk 0.2.10 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0187b06702508e3aa734155601f2ac61bfa9456
-  data.tar.gz: 0978a2745e336986fc17c57796a77f271439ba4a
+  metadata.gz: 4bbd52d83f9670fdf5c7bad8bb916bbce2d2539c
+  data.tar.gz: 3ede177aa9ffe5d102a6bb68e62a256a209c68d6
 SHA512:
-  metadata.gz: 5b19b4372816708a26d2110a823a3fba165fd066ab650f85002d3751ae9efb3f3b3e6eb9163f2a1eca976f9c31ccae00edf760da7216a6c017d8f78a775abb79
-  data.tar.gz: 374a88e38e1ab5469f66500aab6fd2be1f9fead2ea21358698ad8458d5ea26df88eea4ae4f7fdfffe9cedb4b7e8f0b7cedc91bbe930691ebc666414c559c902e
+  metadata.gz: 8fe16f902ee028b8f13237f4bd8784ded62e96df33527da4459cd59044212193d1994b8d8129b8a008e7bb76ec099e04a69e8fc8fd7e4134889c65593f53fb13
+  data.tar.gz: 08d9ada788c89cb5eb1beabd3462b7a1731965e2ecd4171b4bc781de3621a1ac95ab7d4fa1b5b935a5d678b63346f37be46b83950a555f14fc885044cff45a44

data/.ruby-version

@@ -1 +1 @@
-jruby-9.0.4.0
+jruby-9.1.5.0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.11 - 2018-15-26
+* [enhancement] Rate limit is fail fast instead of retry for guess/preview mode
+
 ## 0.2.10 - 2018-03-26
 * [enhancement] Add Zendesk market place header
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.2.10"
+  spec.version       = "0.2.11"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -130,7 +130,7 @@ module Embulk
 
         def export_parallel(path, key, start_time = 0, &block)
           per_page = 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
-          first_response = request(path, per_page: per_page, page: 1)
+          first_response = request(path, false, per_page: per_page, page: 1)
           first_fetched = JSON.parse(first_response.body)
           total_count = first_fetched["count"]
           last_page_num = (total_count / per_page.to_f).ceil
@@ -143,7 +143,7 @@ module Embulk
           execute_thread_pool do |pool|
             (2..last_page_num).each do |page|
               pool.post do
-                response = request(path, per_page: per_page, page: page)
+                response = request(path, false, per_page: per_page, page: page)
                 fetched_records = extract_records_from_response(response, key)
                 Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
                 fetched_records.uniq { |r| r['id'] }.each do |record|
@@ -160,7 +160,7 @@ module Embulk
           per_page = PARTIAL_RECORDS_SIZE
           Embulk.logger.info("Fetching #{path} with page=#{page} (partial)")
 
-          response = request(path, per_page: per_page, page: page)
+          response = request(path, true, per_page: per_page, page: page)
 
           begin
             data = JSON.parse(response.body)
@@ -186,7 +186,7 @@ module Embulk
           execute_thread_pool do |pool|
             loop do
               start_fetching = Time.now
-              response = request(path, query.merge(start_time: start_time))
+              response = request(path, false, query.merge(start_time: start_time))
               actual_fetched = 0
               data = JSON.parse(response.body)
               # no key found in response occasionally => retry
@@ -261,7 +261,7 @@ module Embulk
           httpclient
         end
 
-        def request(path, query = {})
+        def request(path, partial = false, query = {})
           u = URI.parse(config[:login_url])
           u.path = path
 
@@ -277,8 +277,7 @@ module Embulk
           retryer.with_retry do
             Embulk.logger.debug "Fetching #{u.to_s}"
             response = httpclient.get(u.to_s, query, extheader)
-
-            handle_response(response.status, response.headers, response.body)
+            handle_response(response.status, response.headers, response.body, partial)
             response
           end
         end
@@ -312,8 +311,7 @@ module Embulk
                   next
                 end
               end
-              handle_response(message.status, message.headers, chunk)
-
+              handle_response(message.status, message.headers, chunk, true)
               buf << chunk
               break if buf.bytesize > PARTIAL_RECORDS_BYTE_SIZE
             end
@@ -349,13 +347,18 @@ module Embulk
           result
         end
 
-        def wait_rate_limit(retry_after)
-          Embulk.logger.warn "Rate Limited. Waiting #{retry_after} seconds to retry"
-          sleep retry_after
-          throw :retry
+        def wait_rate_limit(retry_after, partial = false)
+          # Won't retry for preview/guess mode
+          if partial
+            raise Embulk::DataError.new("Rate Limited. Waiting #{retry_after} seconds to re-run")
+          else
+            Embulk.logger.warn "Rate Limited. Waiting #{retry_after} seconds to retry"
+            sleep retry_after
+            throw :retry
+          end
         end
 
-        def handle_response(status_code, headers, body)
+        def handle_response(status_code, headers, body, partial = false)
           # https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
           case status_code
           when 200, 404
@@ -379,7 +382,7 @@ module Embulk
           when 429
             # rate limit
             retry_after = headers["Retry-After"]
-            wait_rate_limit(retry_after.to_i)
+            wait_rate_limit(retry_after.to_i, partial)
           when 400..500
             # Won't retry for 4xx range errors except above. Almost they should be ConfigError e.g. 403 Forbidden
             raise Embulk::ConfigError.new("[#{status_code}] #{body}")
@@ -387,7 +390,7 @@ module Embulk
             # 503 is possible rate limit
             retry_after = headers["Retry-After"]
             if retry_after
-              wait_rate_limit(retry_after.to_i)
+              wait_rate_limit(retry_after.to_i, partial)
             else
               raise "[#{status_code}] temporally failure."
             end

data/test/embulk/input/zendesk/test_client.rb

@@ -482,6 +482,14 @@ module Embulk
             stub_response(429, ["Retry-After: #{after}"])
             mock(client).sleep after.to_i
             assert_throw(:retry) do
+              client.tickets(false, &proc{})
+            end
+          end
+
+          test "429 guess/preview fail fast" do
+            after = "123"
+            stub_response(429, ["Retry-After: #{after}"])
+            assert_raise(DataError.new("Rate Limited. Waiting #{after} seconds to re-run")) do
               client.tickets(&proc{})
             end
           end
@@ -505,6 +513,14 @@ module Embulk
             stub_response(503, ["Retry-After: #{after}"])
             mock(client).sleep after.to_i
             assert_throw(:retry) do
+              client.tickets(false, &proc{})
+            end
+          end
+
+          test "503 with Retry-After guess/preview fail fast" do
+            after = "123"
+            stub_response(503, ["Retry-After: #{after}"])
+            assert_raise(DataError.new("Rate Limited. Waiting #{after} seconds to re-run")) do
               client.tickets(&proc{})
             end
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors:
 - uu59
@@ -255,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.6
 signing_key:
 specification_version: 4
 summary: Zendesk input plugin for Embulk