embulk-input-zendesk 0.1.13 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a391e53c7f33423839b92cfbaed3b80765ec8fb
-  data.tar.gz: 54253ba9242ad460cb0252912b928aaf81a77af6
+  metadata.gz: 8965b15e24973482e7f636c44633ace2821f9c0d
+  data.tar.gz: 1bbcaf319cabe8f53ddfde5b2cdeb16f0c54c612
 SHA512:
-  metadata.gz: a0909371764ebeec9b3b083040cab4a6626391d9eb258292428040ce7cc2d91480a2e2edaca6b173f0b8773d3380a21e2e0ca8d462b69f714b25e9bec4242854
-  data.tar.gz: 6f6f530ab66122a1ab329c3a228ee699052acf7114448ef2d4bfe43c11d03dd8afed19ec220a2a23a942178fd380f8c718228716b9ae5dc58f0568949096ce2f
+  metadata.gz: 79ea61da70c871ea6d1bde7789dbc0af0fce75101aa29ac8d7821b0bf73b42df87c925c40dc796a4333cda32d17546ef86c233a463b5d3eed86211cedd2d6e0b
+  data.tar.gz: 5a767959bf7955ed241d67dc5993731046c1abfa328af9738bff8dda40bf16543aa5ec568cb5ec3fcb26912a4790baab88da09ec6f5a5782eddd8bf8b4833aa5

data/CHANGELOG.md

@@ -1,3 +1,31 @@
+## 0.2.6 - 2017-05-23
+* [enhancement] Enable incremental loading for ticket_metrics
+
+## 0.2.5 - 2017-04-21
+* [enhancement] Replace `thread` gem with `concurrent-ruby`, it's more robust and has better queue management [#33](https://github.com/treasure-data/embulk-input-zendesk/pull/33)
+
+## 0.2.4 - 2017-04-20
+* [fixed] Fix thread pool bottleneck [#31](https://github.com/treasure-data/embulk-input-zendesk/pull/31)
+
+## 0.2.3 - 2017-04-18
+* [enhancement] Add flushing to `page_builder` [#29](https://github.com/treasure-data/embulk-input-zendesk/pull/29)
+
+## 0.2.2 - 2017-04-14
+* [enhancement] Improve `httpclient` usage: re-use client instance [#27](https://github.com/treasure-data/embulk-input-zendesk/pull/27)
+
+## 0.2.1 - 2017-04-11
+* [fixed] Mem leak in `export_parallel()` method [#25](https://github.com/treasure-data/embulk-input-zendesk/pull/25)
+
+## 0.2.0 - 2017-04-07
+* [fixed] `time_metrics` is cutoff (archived), need to compare with list of all `tickets` [#23](https://github.com/treasure-data/embulk-input-zendesk/pull/23)
+* [enhancement] Switch to thread pool for `export_parallel()` method [#23](https://github.com/treasure-data/embulk-input-zendesk/pull/23)
+
+## 0.1.15 - 2017-03-30
+* [fixed] Rename JRuby thread, to not expose runtime path [#21](https://github.com/treasure-data/embulk-input-zendesk/pull/21)
+
+## 0.1.14 - 2017-03-28
+* [enhancement] Concurrent fetching base target and related objects [#19](https://github.com/treasure-data/embulk-input-zendesk/pull/19)
+
 ## 0.1.13 - 2017-03-23
 * [fixed] Fix to generate config_diff when no data fetched [#18](https://github.com/treasure-data/embulk-input-zendesk/pull/18)
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.13"
+  spec.version       = "0.2.6"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'perfect_retry', '~> 0.5'
   spec.add_dependency 'httpclient'
+  spec.add_dependency 'concurrent-ruby'
   spec.add_development_dependency 'embulk', ['~> 0.8.1']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/input/zendesk/client.rb

@@ -1,6 +1,6 @@
 require "strscan"
-require "thread"
 require "httpclient"
+require 'concurrent'
 
 module Embulk
   module Input
@@ -10,8 +10,8 @@ module Embulk
 
         PARTIAL_RECORDS_SIZE = 50
         PARTIAL_RECORDS_BYTE_SIZE = 50000
-        AVAILABLE_INCREMENTAL_EXPORT = %w(tickets users organizations ticket_events).freeze
-        UNAVAILABLE_INCREMENTAL_EXPORT = %w(ticket_fields ticket_forms ticket_metrics).freeze
+        AVAILABLE_INCREMENTAL_EXPORT = %w(tickets users organizations ticket_events ticket_metrics).freeze
+        UNAVAILABLE_INCREMENTAL_EXPORT = %w(ticket_fields ticket_forms).freeze
         AVAILABLE_TARGETS = AVAILABLE_INCREMENTAL_EXPORT + UNAVAILABLE_INCREMENTAL_EXPORT
 
         def initialize(config)
@@ -19,10 +19,25 @@ module Embulk
         end
 
         def httpclient
-          httpclient = HTTPClient.new
-          httpclient.connect_timeout = 240 # default:60 is not enough for huge data
-          # httpclient.debug_dev = STDOUT
-          return set_auth(httpclient)
+          # multi-threading + retry can create lot of instances, and each will keep connecting
+          # re-using instance in multi threads can help to omit cleanup code
+          @httpclient ||=
+            begin
+              clnt = HTTPClient.new
+              clnt.connect_timeout = 240 # default:60 is not enough for huge data
+              clnt.receive_timeout = 240 # better change default receive_timeout too
+              # httpclient.debug_dev = STDOUT
+              set_auth(clnt)
+            end
+        end
+
+        def get_pool
+          Concurrent::ThreadPoolExecutor.new(
+            min_threads: 10,
+            max_threads: 100,
+            max_queue: 10_000,
+            fallback_policy: :caller_runs
+          )
         end
 
         def validate_config
@@ -69,6 +84,24 @@ module Embulk
           end
         end
 
+        # Ticket metrics will need to be export using both the non incremental and incremental on ticket
+        # We provide support by filter out ticket_metrics with created at smaller than start time
+        # while passing the incremental start time to the incremental ticket/ticket_metrics export
+        %w(ticket_metrics).each do |target|
+          define_method(target) do |partial = true, start_time = 0, &block|
+            path = "/api/v2/incremental/tickets.json"
+            if partial
+              path = "/api/v2/#{target}.json"
+              # If partial export then we need to use the old end point. Since new end point return both ticket and
+              # ticket metric with ticket come first so the current approach that cut off the response packet won't work
+              # Since partial is only use for preview and guess so this should be fine
+              export(path, target, &block)
+            else
+              incremental_export(path, "metric_sets", start_time, [], partial,{include: "metric_sets"}, &block)
+            end
+          end
+        end
+
         # they have non-incremental API only
         UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
@@ -76,7 +109,7 @@ module Embulk
             if partial
               export(path, target, &block)
             else
-              export_parallel(path, target, &block)
+              export_parallel(path, target, start_time, &block)
             end
           end
         end
@@ -96,7 +129,7 @@ module Embulk
 
         private
 
-        def export_parallel(path, key, workers = 5, &block)
+        def export_parallel(path, key, start_time = 0, &block)
           per_page = 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
           first_response = request(path, per_page: per_page, page: 1)
           first_fetched = JSON.parse(first_response.body)
@@ -104,42 +137,26 @@ module Embulk
           last_page_num = (total_count / per_page.to_f).ceil
           Embulk.logger.info "#{key} records=#{total_count} last_page=#{last_page_num}"
 
-          queue = Queue.new
-          (2..last_page_num).each do |page|
-            queue << page
+          first_fetched[key].uniq { |r| r['id'] }.each do |record|
+            block.call record
+            # known_ticket_ids: collect fetched ticket IDs, to exclude in next step
           end
-          records = first_fetched[key]
-
-          mutex = Mutex.new
-          threads = workers.times.map do |n|
-            Thread.start do
-              loop do
-                break if queue.empty?
-                current_page = nil
-
-                begin
-                  Timeout.timeout(0.1) do
-                    # Somehow queue.pop(true) blocks... timeout is workaround for that
-                    current_page = queue.pop(true)
-                  end
-                rescue Timeout::Error, ThreadError => e
-                  break #=> ThreadError: queue empty
-                end
 
-                response = request(path, per_page: per_page, page: current_page)
-                fetched_records = extract_records_from_response(response, key)
-                mutex.synchronize do
-                  Embulk.logger.info "Fetched #{key} on page=#{current_page}"
-                  records.concat fetched_records
-                end
+          pool = get_pool
+          (2..last_page_num).each do |page|
+            pool.post do
+              response = request(path, per_page: per_page, page: page)
+              fetched_records = extract_records_from_response(response, key)
+              Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
+              fetched_records.uniq { |r| r['id'] }.each do |record|
+                block.call record
               end
             end
           end
-          threads.each(&:join)
 
-          records.uniq {|r| r["id"]}.each do |record|
-            block.call record
-          end
+          pool.shutdown
+          pool.wait_for_termination
+
           nil # this is necessary different with incremental_export
         end
 
@@ -160,18 +177,19 @@ module Embulk
           end
         end
 
-        def incremental_export(path, key, start_time = 0, known_ids = [], partial = true, &block)
+        def incremental_export(path, key, start_time = 0, known_ids = [], partial = true,query = {}, &block)
           if partial
-            records = request_partial(path, {start_time: start_time}).first(5)
+            records = request_partial(path, query.merge({start_time: start_time})).first(5)
             records.uniq{|r| r["id"]}.each do |record|
               block.call record
             end
             return
           end
 
-          loop do
+          pool = get_pool
+          last_data = loop do
             start_fetching = Time.now
-            response = request(path, {start_time: start_time})
+            response = request(path, query.merge({start_time: start_time}))
             begin
               data = JSON.parse(response.body)
             rescue => e
@@ -198,7 +216,7 @@ module Embulk
               next if known_ids.include?(record["id"])
 
               known_ids << record["id"]
-              block.call record
+              pool.post { yield(record) }
               actual_fetched += 1
             end
             Embulk.logger.info "Fetched #{actual_fetched} records from start_time:#{start_time} (#{Time.at(start_time)}) within #{Time.now.to_i - start_fetching.to_i} seconds"
@@ -209,6 +227,10 @@ module Embulk
             #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
             break data if data["count"] < 1000
           end
+
+          pool.shutdown
+          pool.wait_for_termination
+          last_data
         end
 
         def extract_records_from_response(response, key)
@@ -364,6 +386,7 @@ module Embulk
             raise "Server returns unknown status code (#{status_code}) #{body}"
           end
         end
+
       end
     end
   end

data/lib/embulk/input/zendesk/plugin.rb

@@ -1,4 +1,4 @@
-require "perfect_retry"
+require 'perfect_retry'
 
 module Embulk
   module Input
@@ -110,11 +110,14 @@ module Embulk
             args << @start_time.to_i
           end
 
+          mutex = Mutex.new
           fetching_start_at = Time.now
           last_data = client.public_send(method, *args) do |record|
             record = fetch_related_object(record)
             values = extract_values(record)
-            page_builder.add(values)
+            mutex.synchronize do
+              page_builder.add(values)
+            end
             break if preview? # NOTE: preview take care only 1 record. subresources fetching is slow.
           end
           page_builder.finish
@@ -133,14 +136,15 @@ module Embulk
             end
           end
 
-          return task_report
+          task_report
         end
 
         private
 
         def fetch_related_object(record)
-          (task[:includes] || []).each do |ent|
-            record[ent] = client.fetch_subresource(record["id"], task[:target], ent)
+          return record unless task[:includes] && !task[:includes].empty?
+          task[:includes].each do |ent|
+            record[ent] = client.fetch_subresource(record['id'], task[:target], ent)
           end
           record
         end

data/test/embulk/input/zendesk/test_client.rb

@@ -6,7 +6,7 @@ require "embulk/input/zendesk"
 require "override_assert_raise"
 require "fixture_helper"
 require "capture_io"
-
+require "concurrent/atomic/atomic_fixnum"
 module Embulk
   module Input
     module Zendesk
@@ -49,7 +49,40 @@ module Embulk
             end
           end
         end
-
+        sub_test_case "ticket_metrics incremental export" do
+          def client
+            @client ||= Client.new(login_url: login_url, auth_method: "oauth", access_token: access_token, retry_limit: 1, retry_initial_wait_sec: 0)
+          end
+          setup do
+            stub(Embulk).logger { Logger.new(File::NULL) }
+            @httpclient = client.httpclient
+            stub(client).httpclient { @httpclient }
+          end
+          test "fetch ticket_metrics with start_time set" do
+            records = 100.times.map{|n| {"id"=> n, "ticket_id"=>n+1}}
+            start_time = 1488535542
+            @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                    metric_sets: records,
+                    count: records.size,
+                    next_page: nil,
+                }.to_json
+            ].join("\r\n")
+            # lock = Mutex.new
+            result_array = records
+            counter=Concurrent::AtomicFixnum.new(0)
+            handler = proc { |record|
+              assert_include(result_array,record)
+              counter.increment
+            }
+            proxy(@httpclient).get("#{login_url}/api/v2/incremental/tickets.json", {:include => "metric_sets", :start_time => start_time}, anything)
+            client.ticket_metrics(false, start_time, &handler)
+            assert_equal(counter.value, result_array.size)
+          end
+        end
         sub_test_case "ticket_metrics (non-incremental export)" do
           sub_test_case "partial" do
             def client
@@ -73,7 +106,8 @@ module Embulk
                 "",
                 {
                   ticket_metrics: records,
-                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
+                  next_page: "https://treasuredata.zendesk.com/api/v2/incremental/tickets
+.json?include=metric_sets&start_time=1488535542",
                 }.to_json
               ].join("\r\n")
 
@@ -97,18 +131,20 @@ module Embulk
             end
 
             test "fetch ticket_metrics all page" do
-              records = 100.times.map{|n| {"id"=> n}}
+              records = 100.times.map{|n| {"id"=> n, "ticket_id"=>n+1}}
               second_results = [
-                {"id" => 101}
+                {"id" => 101, "ticket_id" => 101}
               ]
+              end_time = 1488535542
               @httpclient.test_loopback_http_response << [
                 "HTTP/1.1 200",
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: records,
-                  count: records.length + second_results.length,
-                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
+                  metric_sets: records,
+                  count: 1000,
+                  next_page: "#{login_url}/api/v2/incremental/tickets.json?include=metric_sets&start_time=1488535542",
+                  end_time: end_time
                 }.to_json
               ].join("\r\n")
 
@@ -117,55 +153,59 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: second_results,
-                  count: records.length + second_results.length,
+                  metric_sets: second_results,
+                  count: second_results.size,
                   next_page: nil,
                 }.to_json
               ].join("\r\n")
+              # lock = Mutex.new
+              result_array = records + second_results
+              counter=Concurrent::AtomicFixnum.new(0)
+              handler = proc { |record|
+                  assert_include(result_array,record)
+                  counter.increment
+              }
+
+              proxy(@httpclient).get("#{login_url}/api/v2/incremental/tickets.json", {:include => "metric_sets", :start_time => 0}, anything)
+              proxy(@httpclient).get("#{login_url}/api/v2/incremental/tickets.json", {:include => "metric_sets",:start_time => end_time}, anything)
 
-              handler = proc { }
-              records.each do |record|
-                mock(handler).call(record)
-              end
-              second_results.each do |record|
-                mock(handler).call(record)
-              end
               client.ticket_metrics(false, &handler)
+              assert_equal(counter.value, result_array.size)
             end
 
-            test "fetch tickets without duplicated" do
+            test "fetch tickets metrics without duplicated" do
               records = [
-                {"id" => 1},
-                {"id" => 2},
-                {"id" => 1},
-                {"id" => 1},
+                {"id" => 1, "ticket_id" => 100},
+                {"id" => 2, "ticket_id" => 200},
+                {"id" => 1, "ticket_id" => 100},
+                {"id" => 1, "ticket_id" => 100},
               ]
               @httpclient.test_loopback_http_response << [
                 "HTTP/1.1 200",
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: records,
+                  metric_sets: records,
                   count: records.length,
                 }.to_json
               ].join("\r\n")
-
-              handler = proc { }
-              mock(handler).call(anything).twice
+              counter = Concurrent::AtomicFixnum.new(0)
+              handler = proc {counter.increment}
               client.ticket_metrics(false, &handler)
+              assert_equal(2,counter.value)
             end
 
-            test "fetch tickets with next_page" do
-              end_time = Time.now.to_i
-
+            test "fetch ticket_metrics with next_page" do
+              end_time = 1488535542
               response_1 = [
                 "HTTP/1.1 200",
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: 100.times.map{|n| {"id" => n}},
-                  count: 101,
-                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
+                  metric_sets: 100.times.map{|n| {"id" => n, "ticket_id" => n+1}},
+                  count: 1001,
+                  end_time: end_time,
+                  next_page: "#{login_url}/api/v2/incremental/tickets.json?include=metric_sets&start_time=1488535542",
                 }.to_json
               ].join("\r\n")
 
@@ -174,17 +214,20 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: [{"id" => 101}],
+                  metric_sets: [{"id" => 101, "ticket_id" => 101}],
                   count: 101,
                 }.to_json
               ].join("\r\n")
 
+
               @httpclient.test_loopback_http_response << response_1
               @httpclient.test_loopback_http_response << response_2
-
-              handler = proc { }
-              mock(handler).call(anything).times(101)
+              counter = Concurrent::AtomicFixnum.new(0)
+              handler = proc { counter.increment }
+              proxy(@httpclient).get("#{login_url}/api/v2/incremental/tickets.json",{:include=>"metric_sets", :start_time=>0},anything)
+              proxy(@httpclient).get("#{login_url}/api/v2/incremental/tickets.json",{:include=>"metric_sets", :start_time=>end_time},anything)
               client.ticket_metrics(false, &handler)
+              assert_equal(101, counter.value)
             end
 
             test "raise DataError when invalid JSON response" do
@@ -232,7 +275,8 @@ module Embulk
             end
 
             test "invoke export when partial=false" do
-              mock(client).export_parallel(anything, "ticket_fields")
+              # Added default `start_time`
+              mock(client).export_parallel(anything, "ticket_fields", 0)
               client.ticket_fields(false)
             end
           end
@@ -244,7 +288,8 @@ module Embulk
             end
 
             test "invoke export when partial=false" do
-              mock(client).export_parallel(anything, "ticket_forms")
+              # Added default `start_time`
+              mock(client).export_parallel(anything, "ticket_forms", 0)
               client.ticket_forms(false)
             end
           end
@@ -520,6 +565,13 @@ module Embulk
           end
         end
 
+        sub_test_case "should not create new instance of httpclient" do
+          test "not create new instance when re-call" do
+            client = Client.new(login_url: login_url, auth_method: "token", username: username, token: token)
+            assert client.httpclient == client.httpclient
+          end
+        end
+
         def login_url
           "http://example.com"
         end

data/test/embulk/input/zendesk/test_plugin.rb

@@ -549,6 +549,43 @@ module Embulk
             end
           end
 
+          sub_test_case "flush each 10k records" do
+            setup do
+              stub(Embulk).logger { Logger.new(File::NULL) }
+              stub(@plugin).preview? { false }
+              @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  tickets: (1..20000).map { |i| { 'id' => i } },
+                  count: 20000,
+                  end_time: 0,
+                }.to_json
+              ].join("\r\n")
+              # to stop pagination (count < 1000)
+              @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  tickets: [{ 'id' => 20001 }],
+                  count: 1,
+                  end_time: 0,
+                }.to_json
+              ].join("\r\n")
+            end
+
+            test "flush is called twice" do
+              omit("This test is no longer valid, flushing is removed now")
+              mock(page_builder).add(anything).times(20001)
+              mock(page_builder).flush.times(2)
+              mock(page_builder).finish
+
+              @plugin.run
+            end
+          end
+
         end
 
         def yml

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.2.6
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-23 00:00:00.000000000 Z
+date: 2017-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: