embulk-input-zendesk 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c154709e56d9966d5bfb53da83c905151afb506
-  data.tar.gz: 744915c536f9fe8c6fa1fd20fd28a11bc57dee69
+  metadata.gz: 73115292227dc5474093e998c1025f4a0fa55c22
+  data.tar.gz: 23cd509f73bfa2c8ccff24478b258295c32c8806
 SHA512:
-  metadata.gz: fcdf6b6607ef344697eb8ea12e1669752821c853dfaf2f5cb675777664ad78e521223a68be813ac202bd884a398f9d194e9fbc1eaec9a9c0b01cb38e2948336d
-  data.tar.gz: b1bd6a19d263d48efdf9eb4047479dbce78e204e9d0304628d1fcc7e5670a01b3dbcdea20c9e5a8774c7068626c1b441185333a5c51b5986375bad188933f88c
+  metadata.gz: 6364750da3c79b7dc8a8e9507887884d3cda61445ad8f775b7bffbfb764f07a49938621380060c323c43a1a806b4ffc20f1d69a3e8e045ec7c12aa0e8d1c84a3
+  data.tar.gz: 93ae137dacd61c9b56f323916f6993a6f7b077231646227e85003c85d1966ee8b66c1ab99cf87c922ccc55f021ff5edeb8f136aad5039448a6d99fc5af98961d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.7 - 2016-06-04
+* [enhancement] Improvements for non incremental export [#12](https://github.com/treasure-data/embulk-input-zendesk/pull/12)
+
 ## 0.1.6 - 2016-05-09
 * [fixed] Fix non-incremental export to fetch all records [#11](https://github.com/treasure-data/embulk-input-zendesk/pull/11)
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.6"
+  spec.version       = "0.1.7"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -1,4 +1,5 @@
 require "strscan"
+require "thread"
 require "httpclient"
 
 module Embulk
@@ -71,11 +72,16 @@ module Embulk
         UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
             path = "/api/v2/#{target}.json"
-            export(path, target, partial, &block)
+            if partial
+              export(path, target, &block)
+            else
+              export_parallel(path, target, &block)
+            end
           end
         end
 
         def fetch_subresource(record_id, base, target)
+          Embulk.logger.info "Fetching subresource #{target} of #{base}:#{record_id}"
           response = request("/api/v2/#{base}/#{record_id}/#{target}.json")
           return [] if response.status == 404
 
@@ -89,9 +95,56 @@ module Embulk
 
         private
 
-        def export(path, key, partial, page = 1, known_ids = [], &block)
-          per_page = partial ? PARTIAL_RECORDS_SIZE : 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
-          Embulk.logger.debug("#{path} with page=#{page}" + (partial ? " (partial)" : ""))
+        def export_parallel(path, key, workers = 5, &block)
+          per_page = 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
+          first_response = request(path, per_page: per_page, page: 1)
+          first_fetched = JSON.parse(first_response.body)
+          total_count = first_fetched["count"]
+          last_page_num = (total_count / per_page.to_f).ceil
+          Embulk.logger.info "#{key} records=#{total_count} last_page=#{last_page_num}"
+
+          queue = Queue.new
+          (2..last_page_num).each do |page|
+            queue << page
+          end
+          records = first_fetched[key]
+
+          mutex = Mutex.new
+          threads = workers.times.map do |n|
+            Thread.start do
+              loop do
+                break if queue.empty?
+                current_page = nil
+
+                begin
+                  Timeout.timeout(0.1) do
+                    # Somehow queue.pop(true) blocks... timeout is workaround for that
+                    current_page = queue.pop(true)
+                  end
+                rescue Timeout::Error, ThreadError => e
+                  break #=> ThreadError: queue empty
+                end
+
+                response = request(path, per_page: per_page, page: current_page)
+                fetched_records = extract_records_from_response(response, key)
+                mutex.synchronize do
+                  Embulk.logger.info "Fetched #{key} on page=#{current_page}"
+                  records.concat fetched_records
+                end
+              end
+            end
+          end
+          threads.each(&:join)
+
+          records.uniq {|r| r["id"]}.each do |record|
+            block.call record
+          end
+          nil # this is necessary different with incremental_export
+        end
+
+        def export(path, key, page = 1, &block)
+          per_page = PARTIAL_RECORDS_SIZE
+          Embulk.logger.info("Fetching #{path} with page=#{page} (partial)")
 
           response = request(path, per_page: per_page, page: page)
 
@@ -102,18 +155,8 @@ module Embulk
           end
 
           data[key].each do |record|
-            next if known_ids.include?(record["id"])
-            known_ids << record["id"]
-
             block.call record
           end
-          return if partial
-
-          if data["next_page"]
-            return export(path, key, partial, page + 1, &block)
-          end
-
-          nil # this is necessary different with incremental_export
         end
 
         def incremental_export(path, key, start_time = 0, known_ids = [], partial = true, &block)
@@ -126,7 +169,7 @@ module Embulk
             rescue => e
               raise Embulk::DataError.new(e)
             end
-            Embulk.logger.debug "start_time:#{start_time} (#{Time.at(start_time)}) count:#{data["count"]} next_page:#{data["next_page"]} end_time:#{data["end_time"]} "
+            Embulk.logger.info "Fetched records from #{start_time} (#{Time.at(start_time)})"
             records = data[key]
           end
 
@@ -151,6 +194,15 @@ module Embulk
           end
         end
 
+        def extract_records_from_response(response, key)
+          begin
+            data = JSON.parse(response.body)
+            data[key]
+          rescue => e
+            raise Embulk::DataError.new(e)
+          end
+        end
+
         def retryer
           PerfectRetry.new do |config|
             config.limit = @config[:retry_limit]

data/lib/embulk/input/zendesk/plugin.rb

@@ -82,7 +82,7 @@ module Embulk
             access_token: config.param("access_token", :string, default: nil),
             start_time: config.param("start_time", :string, default: nil),
             retry_limit: config.param("retry_limit", :integer, default: 5),
-            retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
+            retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 4),
             incremental: config.param("incremental", :bool, default: true),
             schema: config.param(:columns, :array, default: []),
             includes: config.param(:includes, :array, default: []),

data/test/embulk/input/zendesk/test_client.rb

@@ -97,12 +97,9 @@ module Embulk
             end
 
             test "fetch ticket_metrics all page" do
-              records = [
-                {"id" => 1},
-                {"id" => 2},
-              ]
+              records = 100.times.map{|n| {"id"=> n}}
               second_results = [
-                {"id" => 3}
+                {"id" => 101}
               ]
               @httpclient.test_loopback_http_response << [
                 "HTTP/1.1 200",
@@ -110,6 +107,7 @@ module Embulk
                 "",
                 {
                   ticket_metrics: records,
+                  count: records.length + second_results.length,
                   next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
                 }.to_json
               ].join("\r\n")
@@ -120,6 +118,7 @@ module Embulk
                 "",
                 {
                   ticket_metrics: second_results,
+                  count: records.length + second_results.length,
                   next_page: nil,
                 }.to_json
               ].join("\r\n")
@@ -146,7 +145,8 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: records
+                  ticket_metrics: records,
+                  count: records.length,
                 }.to_json
               ].join("\r\n")
 
@@ -163,7 +163,8 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: [{"id" => 1}],
+                  ticket_metrics: 100.times.map{|n| {"id" => n}},
+                  count: 101,
                   next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
                 }.to_json
               ].join("\r\n")
@@ -173,8 +174,8 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  ticket_metrics: [{"id" => 2}],
-                  count: 2,
+                  ticket_metrics: [{"id" => 101}],
+                  count: 101,
                 }.to_json
               ].join("\r\n")
 
@@ -182,7 +183,7 @@ module Embulk
               @httpclient.test_loopback_http_response << response_2
 
               handler = proc { }
-              mock(handler).call(anything).twice
+              mock(handler).call(anything).times(101)
               client.ticket_metrics(false, &handler)
             end
 
@@ -226,24 +227,24 @@ module Embulk
 
           sub_test_case "ticket_fields" do
             test "invoke export when partial=true" do
-              mock(client).export(anything, "ticket_fields", anything)
+              mock(client).export(anything, "ticket_fields")
               client.ticket_fields(true)
             end
 
             test "invoke export when partial=false" do
-              mock(client).export(anything, "ticket_fields", anything)
+              mock(client).export_parallel(anything, "ticket_fields")
               client.ticket_fields(false)
             end
           end
 
           sub_test_case "ticket_forms" do
             test "invoke export when partial=true" do
-              mock(client).export(anything, "ticket_forms", anything)
+              mock(client).export(anything, "ticket_forms")
               client.ticket_forms(true)
             end
 
             test "invoke export when partial=false" do
-              mock(client).export(anything, "ticket_forms", anything)
+              mock(client).export_parallel(anything, "ticket_forms")
               client.ticket_forms(false)
             end
           end

data/test/embulk/input/zendesk/test_plugin.rb

@@ -353,6 +353,7 @@ module Embulk
           sub_test_case "run" do
             setup do
               stub(@plugin).preview? { false }
+              stub(Embulk).logger { Logger.new(File::NULL) }
             end
 
             test "call ticket_all method instead of tickets" do
@@ -416,6 +417,7 @@ module Embulk
                 ].join("\r\n")
                 stub(page_builder).add(anything)
                 stub(page_builder).finish
+                stub(Embulk).logger { Logger.new(File::NULL) }
               end
 
               sub_test_case "incremental: true" do
@@ -443,6 +445,7 @@ module Embulk
 
             sub_test_case "casting value" do
               setup do
+                stub(Embulk).logger { Logger.new(File::NULL) }
                 stub(@plugin).preview? { false }
                 @httpclient.test_loopback_http_response << [
                   "HTTP/1.1 200",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement