embulk-input-zendesk 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8965b15e24973482e7f636c44633ace2821f9c0d
-  data.tar.gz: 1bbcaf319cabe8f53ddfde5b2cdeb16f0c54c612
+  metadata.gz: 3d1102bce0d8a2464284659c4349277604022de6
+  data.tar.gz: 44688236a95097808b51dfaa2528c87ba3230f6f
 SHA512:
-  metadata.gz: 79ea61da70c871ea6d1bde7789dbc0af0fce75101aa29ac8d7821b0bf73b42df87c925c40dc796a4333cda32d17546ef86c233a463b5d3eed86211cedd2d6e0b
-  data.tar.gz: 5a767959bf7955ed241d67dc5993731046c1abfa328af9738bff8dda40bf16543aa5ec568cb5ec3fcb26912a4790baab88da09ec6f5a5782eddd8bf8b4833aa5
+  metadata.gz: 9ee108f8f523bbc57ef69d4f943d51fa2ad1c18db75e91d8433b5a01178bc1f037a45f43db4334f7e0ca3cb8ed350e9d89180f769340eef53e0668a4b86bd81b
+  data.tar.gz: f3ac93d20e468e462a622ece7c2287935c9011739064935aaeccd899b3629aebc91258ee6363d8b1ca111b918a7202484cd21c78b71f4aa412159accd56f62cc

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.7 - 2017-07-19
+* [fixed] Ensure thread pool is shutdown [#38](https://github.com/treasure-data/embulk-input-zendesk/pull/38)
+* [enhancement] Add retry for temporary error: missing required key from JSON response
+
 ## 0.2.6 - 2017-05-23
 * [enhancement] Enable incremental loading for ticket_metrics
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.2.6"
+  spec.version       = "0.2.7"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -31,7 +31,7 @@ module Embulk
             end
         end
 
-        def get_pool
+        def create_pool
           Concurrent::ThreadPoolExecutor.new(
             min_threads: 10,
             max_threads: 100,
@@ -69,36 +69,25 @@ module Embulk
         end
 
         # they have both Incremental API and non-incremental API
-        %w(tickets users organizations).each do |target|
+        # 170717: `ticket_events` can use standard endpoint format now, ie. `<target>.json`
+        %w(tickets ticket_events users organizations).each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
             # Always use incremental_export. There is some difference between incremental_export and export.
             incremental_export("/api/v2/incremental/#{target}.json", target, start_time, [], partial, &block)
           end
         end
 
-        # they have incremental API only
-        %w(ticket_events).each do |target|
-          define_method(target) do |partial = true, start_time = 0, &block|
-            path = "/api/v2/incremental/#{target}"
-            incremental_export(path, target, start_time, [], partial, &block)
-          end
-        end
-
         # Ticket metrics will need to be export using both the non incremental and incremental on ticket
         # We provide support by filter out ticket_metrics with created at smaller than start time
         # while passing the incremental start time to the incremental ticket/ticket_metrics export
-        %w(ticket_metrics).each do |target|
-          define_method(target) do |partial = true, start_time = 0, &block|
-            path = "/api/v2/incremental/tickets.json"
-            if partial
-              path = "/api/v2/#{target}.json"
-              # If partial export then we need to use the old end point. Since new end point return both ticket and
-              # ticket metric with ticket come first so the current approach that cut off the response packet won't work
-              # Since partial is only use for preview and guess so this should be fine
-              export(path, target, &block)
-            else
-              incremental_export(path, "metric_sets", start_time, [], partial,{include: "metric_sets"}, &block)
-            end
+        define_method('ticket_metrics') do |partial = true, start_time = 0, &block|
+          if partial
+            # If partial export then we need to use the old end point. Since new end point return both ticket and
+            # ticket metric with ticket come first so the current approach that cut off the response packet won't work
+            # Since partial is only use for preview and guess so this should be fine
+            export('/api/v2/ticket_metrics.json', 'ticket_metrics', &block)
+          else
+            incremental_export('/api/v2/incremental/tickets.json', 'metric_sets', start_time, [], partial, { include: 'metric_sets' }, &block)
           end
         end
 
@@ -139,24 +128,21 @@ module Embulk
 
           first_fetched[key].uniq { |r| r['id'] }.each do |record|
             block.call record
-            # known_ticket_ids: collect fetched ticket IDs, to exclude in next step
           end
 
-          pool = get_pool
-          (2..last_page_num).each do |page|
-            pool.post do
-              response = request(path, per_page: per_page, page: page)
-              fetched_records = extract_records_from_response(response, key)
-              Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
-              fetched_records.uniq { |r| r['id'] }.each do |record|
-                block.call record
+          execute_thread_pool do |pool|
+            (2..last_page_num).each do |page|
+              pool.post do
+                response = request(path, per_page: per_page, page: page)
+                fetched_records = extract_records_from_response(response, key)
+                Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
+                fetched_records.uniq { |r| r['id'] }.each do |record|
+                  block.call record
+                end
               end
             end
           end
 
-          pool.shutdown
-          pool.wait_for_termination
-
           nil # this is necessary different with incremental_export
         end
 
@@ -177,60 +163,55 @@ module Embulk
           end
         end
 
-        def incremental_export(path, key, start_time = 0, known_ids = [], partial = true,query = {}, &block)
+        def incremental_export(path, key, start_time = 0, known_ids = [], partial = true, query = {}, &block)
+          query.merge!(start_time: start_time)
           if partial
-            records = request_partial(path, query.merge({start_time: start_time})).first(5)
+            records = request_partial(path, query).first(5)
             records.uniq{|r| r["id"]}.each do |record|
               block.call record
             end
             return
           end
 
-          pool = get_pool
-          last_data = loop do
-            start_fetching = Time.now
-            response = request(path, query.merge({start_time: start_time}))
-            begin
+          execute_thread_pool do |pool|
+            loop do
+              start_fetching = Time.now
+              response = request(path, query)
+              actual_fetched = 0
               data = JSON.parse(response.body)
-            rescue => e
-              raise Embulk::DataError.new(e)
-            end
-            actual_fetched = 0
-            records = data[key]
-            records.each do |record|
-              # https://developer.zendesk.com/rest_api/docs/core/incremental_export#excluding-system-updates
-              # "generated_timestamp" will be updated when Zendesk internal changing
-              # "updated_at" will be updated when ticket data was changed
-              # start_time for query parameter will be processed on Zendesk with generated_timestamp,
-              # but it was calculated by record' updated_at time.
-              # So the doesn't changed record from previous import would be appear by Zendesk internal changes.
-              # We ignore record that has updated_at <= start_time
-              if start_time && record["generated_timestamp"] && record["updated_at"]
-                updated_at = Time.parse(record["updated_at"])
-                next if updated_at <= Time.at(start_time)
-              end
+              # no key found in response occasionally => retry
+              raise TempError, "No '#{key}' found in JSON response" unless data.key? key
+              data[key].each do |record|
+                # https://developer.zendesk.com/rest_api/docs/core/incremental_export#excluding-system-updates
+                # "generated_timestamp" will be updated when Zendesk internal changing
+                # "updated_at" will be updated when ticket data was changed
+                # start_time for query parameter will be processed on Zendesk with generated_timestamp,
+                # but it was calculated by record' updated_at time.
+                # So the doesn't changed record from previous import would be appear by Zendesk internal changes.
+                # We ignore record that has updated_at <= start_time
+                if start_time && record["generated_timestamp"] && record["updated_at"]
+                  updated_at = Time.parse(record["updated_at"])
+                  next if updated_at <= Time.at(start_time)
+                end
 
-              # de-duplicated records.
-              # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
-              # https://github.com/zendesk/zendesk_api_client_rb/issues/251
-              next if known_ids.include?(record["id"])
+                # de-duplicated records.
+                # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
+                # https://github.com/zendesk/zendesk_api_client_rb/issues/251
+                next if known_ids.include?(record["id"])
 
-              known_ids << record["id"]
-              pool.post { yield(record) }
-              actual_fetched += 1
-            end
-            Embulk.logger.info "Fetched #{actual_fetched} records from start_time:#{start_time} (#{Time.at(start_time)}) within #{Time.now.to_i - start_fetching.to_i} seconds"
-            start_time = data["end_time"]
+                known_ids << record["id"]
+                pool.post { block.call record }
+                actual_fetched += 1
+              end
+              Embulk.logger.info "Fetched #{actual_fetched} records from start_time:#{start_time} (#{Time.at(start_time)}) within #{Time.now.to_i - start_fetching.to_i} seconds"
+              start_time = data["end_time"]
 
-            # NOTE: If count is less than 1000, then stop paginating.
-            #       Otherwise, use the next_page URL to get the next page of results.
-            #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
-            break data if data["count"] < 1000
+              # NOTE: If count is less than 1000, then stop paginating.
+              #       Otherwise, use the next_page URL to get the next page of results.
+              #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
+              break data if data["count"] < 1000
+            end
           end
-
-          pool.shutdown
-          pool.wait_for_termination
-          last_data
         end
 
         def extract_records_from_response(response, key)
@@ -387,6 +368,27 @@ module Embulk
           end
         end
 
+        def execute_thread_pool(&block)
+          pool = create_pool
+          pr = PerfectRetry.new do |config|
+            config.limit = @config[:retry_limit]
+            config.logger = Embulk.logger
+            config.log_level = nil
+            config.rescues = [TempError]
+            config.sleep = lambda{|n| @config[:retry_initial_wait_sec]* (2 ** (n-1)) }
+          end
+          pr.with_retry { block.call(pool) }
+        rescue => e
+          raise Embulk::DataError.new(e)
+        ensure
+          Embulk.logger.info 'ThreadPool shutting down...'
+          pool.shutdown
+          pool.wait_for_termination
+          Embulk.logger.info "ThreadPool shutdown? #{pool.shutdown?}"
+        end
+      end
+
+      class TempError < StandardError
       end
     end
   end

data/test/embulk/input/zendesk/test_client.rb

@@ -572,6 +572,63 @@ module Embulk
           end
         end
 
+        sub_test_case "ensure thread pool is shutdown with/without errors, retry for TempError" do
+          def client
+            @client ||= Client.new(login_url: login_url, auth_method: "oauth", access_token: access_token, retry_limit: 1, retry_initial_wait_sec: 0)
+          end
+
+          setup do
+            stub(Embulk).logger { Logger.new(File::NULL) }
+            @httpclient = client.httpclient
+            stub(client).httpclient { @httpclient }
+            @pool = Concurrent::ThreadPoolExecutor.new
+            stub(client).create_pool { @pool }
+          end
+          test "should shutdown pool - without error" do
+            @httpclient.test_loopback_http_response << [
+              "HTTP/1.1 200",
+              "Content-Type: application/json",
+              "",
+              {
+                ticket_fields: [{ id: 1 }],
+                count: 1
+              }.to_json
+            ].join("\r\n")
+            handler = proc { }
+            client.ticket_fields(false, &handler)
+            assert_equal(true, @pool.shutdown?)
+          end
+
+          test "should shutdown pool - with TempError (retry)" do
+            response = [
+              "HTTP/1.1 200",
+              "Content-Type: application/json",
+              "",
+              { }.to_json # no required key: `tickets`, raise TempError
+            ].join("\r\n")
+            @httpclient.test_loopback_http_response << response
+            @httpclient.test_loopback_http_response << response # retry 1
+            assert_raise(TempError) do
+              client.tickets(false)
+            end
+            assert_equal(true, @pool.shutdown?)
+          end
+
+          test "should shutdown pool - with DataError (no retry)" do
+            response = [
+              "HTTP/1.1 400", # unhandled error, wrapped in DataError
+              "Content-Type: application/json",
+              "",
+              { }.to_json
+            ].join("\r\n")
+            @httpclient.test_loopback_http_response << response
+            assert_raise(DataError) do
+              client.tickets(false)
+            end
+            assert_equal(true, @pool.shutdown?)
+          end
+        end
+
         def login_url
           "http://example.com"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - uu59
@@ -10,190 +10,190 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-23 00:00:00.000000000 Z
+date: 2017-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: perfect_retry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
-  name: perfect_retry
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: httpclient
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.5'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: httpclient
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: concurrent-ruby
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: embulk
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.8.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.1
-  name: embulk
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: bundler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.1
-- !ruby/object:Gem::Dependency
+        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
+        version: '10.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: pry
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: test-unit
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 3.1.5
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.5
-  name: test-unit
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.5
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: test-unit-rr
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: rr
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 1.1.2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.2
-  name: rr
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: simplecov
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.2
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: simplecov
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: gem_release_helper
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: gem_release_helper
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
+        version: '0.6'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-  name: codeclimate-test-reporter
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
 description: Loads records from Zendesk.
 email:
 - k@uu59.org