embulk-input-zendesk 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdd23bdf56e325f692e469316f07d92a9173727b
-  data.tar.gz: d70d14895b9e26f7f57b03c1b56cb5ac1382b755
+  metadata.gz: 1c154709e56d9966d5bfb53da83c905151afb506
+  data.tar.gz: 744915c536f9fe8c6fa1fd20fd28a11bc57dee69
 SHA512:
-  metadata.gz: 2b6fb9f91fa3d5f58b8c3ad270084dc81c1dd76c14578461932a1f0ad5453f7c05649d4cc62d74c8053c543e084fe6ae2e90871d490591cd6216f3f16aebac7a
-  data.tar.gz: 5ef68347436a5d175377cdf2c1c27ff8cefdce8ee73e2f97c4aae14bd08fa54d180569035aa714b83ee19f37f474147905d28fb7260fbbfd29908623b12d8c82
+  metadata.gz: fcdf6b6607ef344697eb8ea12e1669752821c853dfaf2f5cb675777664ad78e521223a68be813ac202bd884a398f9d194e9fbc1eaec9a9c0b01cb38e2948336d
+  data.tar.gz: b1bd6a19d263d48efdf9eb4047479dbce78e204e9d0304628d1fcc7e5670a01b3dbcdea20c9e5a8774c7068626c1b441185333a5c51b5986375bad188933f88c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.6 - 2016-05-09
+* [fixed] Fix non-incremental export to fetch all records [#11](https://github.com/treasure-data/embulk-input-zendesk/pull/11)
+
 ## 0.1.5 - 2016-04-14
 * [enhancement] Mitigate debug pain when many retry then error [#10](https://github.com/treasure-data/embulk-input-zendesk/pull/10)
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.5"
+  spec.version       = "0.1.6"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -71,7 +71,7 @@ module Embulk
         UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
             path = "/api/v2/#{target}.json"
-            export(path, target, partial ? PARTIAL_RECORDS_SIZE : 1000, &block)
+            export(path, target, partial, &block)
           end
         end
 
@@ -89,11 +89,11 @@ module Embulk
 
         private
 
-        def export(path, key, per_page, &block)
-          # for `embulk guess` and `embulk preview` to fetch ~50 records only.
-          # incremental export API has supported only 1000 per page, it is too large to guess/preview
-          Embulk.logger.debug "#{path} with per_page: #{per_page}"
-          response = request(path, per_page: per_page)
+        def export(path, key, partial, page = 1, known_ids = [], &block)
+          per_page = partial ? PARTIAL_RECORDS_SIZE : 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
+          Embulk.logger.debug("#{path} with page=#{page}" + (partial ? " (partial)" : ""))
+
+          response = request(path, per_page: per_page, page: page)
 
           begin
             data = JSON.parse(response.body)
@@ -102,8 +102,16 @@ module Embulk
           end
 
           data[key].each do |record|
+            next if known_ids.include?(record["id"])
+            known_ids << record["id"]
+
             block.call record
           end
+          return if partial
+
+          if data["next_page"]
+            return export(path, key, partial, page + 1, &block)
+          end
 
           nil # this is necessary different with incremental_export
         end

data/test/embulk/input/zendesk/test_client.rb

@@ -15,7 +15,7 @@ module Embulk
         include FixtureHelper
         include CaptureIo
 
-        sub_test_case "tickets" do
+        sub_test_case "tickets (incremental export)" do
           sub_test_case "partial" do
             def client
               @client ||= Client.new(login_url: login_url, auth_method: "oauth", access_token: access_token, retry_limit: 1, retry_initial_wait_sec: 0)
@@ -48,6 +48,42 @@ module Embulk
               client.tickets(&handler)
             end
           end
+        end
+
+        sub_test_case "ticket_metrics (non-incremental export)" do
+          sub_test_case "partial" do
+            def client
+              @client ||= Client.new(login_url: login_url, auth_method: "oauth", access_token: access_token, retry_limit: 1, retry_initial_wait_sec: 0)
+            end
+
+            setup do
+              stub(Embulk).logger { Logger.new(File::NULL) }
+              @httpclient = client.httpclient
+              stub(client).httpclient { @httpclient }
+            end
+
+            test "fetch ticket_metrics first page only" do
+              records = [
+                {"id" => 1},
+                {"id" => 2},
+              ]
+              @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  ticket_metrics: records,
+                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
+                }.to_json
+              ].join("\r\n")
+
+              handler = proc { }
+              records.each do |record|
+                mock(handler).call(record)
+              end
+              client.ticket_metrics(true, &handler)
+            end
+          end
 
           sub_test_case "all" do
             def client
@@ -60,29 +96,46 @@ module Embulk
               stub(client).httpclient { @httpclient }
             end
 
-            test "fetch tickets" do
-              tickets = [
+            test "fetch ticket_metrics all page" do
+              records = [
                 {"id" => 1},
                 {"id" => 2},
               ]
+              second_results = [
+                {"id" => 3}
+              ]
               @httpclient.test_loopback_http_response << [
                 "HTTP/1.1 200",
                 "Content-Type: application/json",
                 "",
                 {
-                  tickets: tickets
+                  ticket_metrics: records,
+                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
+                }.to_json
+              ].join("\r\n")
+
+              @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  ticket_metrics: second_results,
+                  next_page: nil,
                 }.to_json
               ].join("\r\n")
 
               handler = proc { }
-              tickets.each do |ticket|
-                mock(handler).call(ticket)
+              records.each do |record|
+                mock(handler).call(record)
+              end
+              second_results.each do |record|
+                mock(handler).call(record)
               end
-              client.tickets(false, &handler)
+              client.ticket_metrics(false, &handler)
             end
 
             test "fetch tickets without duplicated" do
-              tickets = [
+              records = [
                 {"id" => 1},
                 {"id" => 2},
                 {"id" => 1},
@@ -93,13 +146,13 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  tickets: tickets
+                  ticket_metrics: records
                 }.to_json
               ].join("\r\n")
 
               handler = proc { }
               mock(handler).call(anything).twice
-              client.tickets(false, &handler)
+              client.ticket_metrics(false, &handler)
             end
 
             test "fetch tickets with next_page" do
@@ -110,9 +163,8 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  tickets: [{"id" => 1}],
-                  count: 1000,
-                  end_time: end_time,
+                  ticket_metrics: [{"id" => 1}],
+                  next_page: "https://treasuredata.zendesk.com/api/v2/ticket_metrics.json?page=2",
                 }.to_json
               ].join("\r\n")
 
@@ -121,7 +173,7 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  tickets: [{"id" => 2}],
+                  ticket_metrics: [{"id" => 2}],
                   count: 2,
                 }.to_json
               ].join("\r\n")
@@ -131,7 +183,7 @@ module Embulk
 
               handler = proc { }
               mock(handler).call(anything).twice
-              client.tickets(false, &handler)
+              client.ticket_metrics(false, &handler)
             end
 
             test "raise DataError when invalid JSON response" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-14 00:00:00.000000000 Z
+date: 2016-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement