embulk-input-zendesk 0.2.13 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0911f65a242c33edcf81953f2546825701b2a4c
-  data.tar.gz: 34b91761ad01712a8ba0bb6a9c258d9555dd5a65
+  metadata.gz: eae8ef806e71aa366b06bc45075e36020c783c11
+  data.tar.gz: d8584f430c482719f7ec2c8ec2c441cfbe7f1a02
 SHA512:
-  metadata.gz: 85e63afc0356899056627249dda852e1a9fe0af9de006335b86d89c01bc53ac97d93a8974f82c27ce69041ad22a40ca271e85d77853d7378cc425198657d6224
-  data.tar.gz: 01e8c2238dd8aa341f3560ed1c8079f0d68454e3a9964137483cedaf625bbec42a4c9bb6ae1cd3fefcdaa9fe4a68896b049150968efe8c4782ac06e943034d88
+  metadata.gz: 293666013500b2074377043e545198e394b3e65876c916630b69cc922ec0639887f4101104e419f5d45f5ed95bc05197b528bb794bb9d2361bfa4fd1a939b6e6
+  data.tar.gz: 9ab732489802ac3cb5ec658785d1d8b09e3331bcf90362165fb3c92477b44ea7f1887aeb47b5e658f12f83b1b3e9c4ed7a813d50fbb9663401a211e9923f3fc9

data/.ruby-version

@@ -1 +1 @@
-jruby-9.1.5.0
+jruby-9.1.17.0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.14 - 2019-01-25
+* [fixed] Disable pagination and add `dedup` option for non-incremental targets [#49](https://github.com/treasure-data/embulk-input-zendesk/pull/49)
+
 ## 0.2.13 - 2019-01-14
 * [enhancement] Add `dedup` option, in order to avoid OOM when importing large dataset [#48](https://github.com/treasure-data/embulk-input-zendesk/pull/48)
 

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.2.13"
+  spec.version       = "0.2.14"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -104,12 +104,12 @@ module Embulk
 
         # they have non-incremental API only
         UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
-          define_method(target) do |partial = true, start_time = 0, &block|
+          define_method(target) do |partial = true, start_time = 0, dedup = true, &block|
             path = "/api/v2/#{target}.json"
             if partial
               export(path, target, &block)
             else
-              export_parallel(path, target, start_time, &block)
+              export_parallel(path, target, start_time, dedup, false, &block)
             end
           end
         end
@@ -129,26 +129,27 @@ module Embulk
 
         private
 
-        def export_parallel(path, key, start_time = 0, &block)
+        def export_parallel(path, key, start_time = 0, dedup = true, paging = true, &block)
           per_page = 100 # 100 is maximum https://developer.zendesk.com/rest_api/docs/core/introduction#pagination
           first_response = request(path, false, per_page: per_page, page: 1)
           first_fetched = JSON.parse(first_response.body)
           total_count = first_fetched["count"]
           last_page_num = (total_count / per_page.to_f).ceil
-          Embulk.logger.info "#{key} records=#{total_count} last_page=#{last_page_num}"
-
-          first_fetched[key].uniq { |r| r['id'] }.each do |record|
-            block.call record
-          end
-
-          execute_thread_pool do |pool|
-            (2..last_page_num).each do |page|
-              pool.post do
-                response = request(path, false, per_page: per_page, page: page)
-                fetched_records = extract_records_from_response(response, key)
-                Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
-                fetched_records.uniq { |r| r['id'] }.each do |record|
-                  block.call record
+          Embulk.logger.info "#{key} records=#{total_count} last_page=#{paging ? last_page_num : 1}"
+
+          handler = lambda { |records| records.each { |r| block.call r } }
+          handler.call(dedup ? first_fetched[key].uniq { |r| r['id'] } : first_fetched[key])
+
+          # stop if endpoints have no pagination, ie. API returns all records
+          # `ticket_fields`, `ticket_forms`
+          if paging
+            execute_thread_pool do |pool|
+              (2..last_page_num).each do |page|
+                pool.post do
+                  response = request(path, false, per_page: per_page, page: page)
+                  fetched_records = extract_records_from_response(response, key)
+                  Embulk.logger.info "Fetched #{key} on page=#{page} >>> size: #{fetched_records.length}"
+                  handler.call(dedup ? fetched_records.uniq { |r| r['id'] } : fetched_records)
                 end
               end
             end

data/test/embulk/input/zendesk/test_client.rb

@@ -107,8 +107,7 @@ module Embulk
                 "",
                 {
                   ticket_metrics: records,
-                  next_page: "https://treasuredata.zendesk.com/api/v2/incremental/tickets
-.json?include=metric_sets&start_time=1488535542",
+                  next_page: "https://treasuredata.zendesk.com/api/v2/incremental/tickets.json?include=metric_sets&start_time=1488535542",
                 }.to_json
               ].join("\r\n")
 
@@ -299,7 +298,7 @@ module Embulk
 
             test "invoke export when partial=false" do
               # Added default `start_time`
-              mock(client).export_parallel(anything, "ticket_fields", 0)
+              mock(client).export_parallel(anything, "ticket_fields", 0, true, false) # new args: `dedup`, `paging`
               client.ticket_fields(false)
             end
           end
@@ -312,10 +311,40 @@ module Embulk
 
             test "invoke export when partial=false" do
               # Added default `start_time`
-              mock(client).export_parallel(anything, "ticket_forms", 0)
+              mock(client).export_parallel(anything, "ticket_forms", 0, true, false) # new args: `dedup`, `paging`)
               client.ticket_forms(false)
             end
           end
+
+          sub_test_case "no pagination" do
+            data("ticket_fields", "ticket_fields")
+            data("ticket_forms", "ticket_forms")
+            test "non-incremental targets" do |target|
+              response = [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  target => 200.times.map{|n| {"id" => n}},
+                  count: 200,
+                  next_page: nil,
+                  previous_page: nil,
+                }.to_json
+              ].join("\r\n")
+
+              # mock multiple responses, to simulate real API behavior
+              @httpclient.test_loopback_http_response << response
+              @httpclient.test_loopback_http_response << response
+              counter = Concurrent::AtomicFixnum.new(0)
+              handler = proc { counter.increment }
+              # validate expected target
+              proxy(@httpclient).get("#{login_url}/api/v2/#{target}.json",anything,anything)
+              # (`partial`, `start_time`, `default`, `block`)
+              client.public_send(target, false, 0, true, &handler)
+              # only ingest 200 records
+              assert_equal(200, counter.value)
+            end
+          end
         end
 
 
@@ -629,12 +658,12 @@ module Embulk
               "Content-Type: application/json",
               "",
               {
-                ticket_fields: [{ id: 1 }],
+                tickets: [{ id: 1 }],
                 count: 1
               }.to_json
             ].join("\r\n")
             handler = proc { }
-            client.ticket_fields(false, &handler)
+            client.tickets(false, &handler)
             assert_equal(true, @pool.shutdown?)
           end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.14
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-14 00:00:00.000000000 Z
+date: 2019-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -261,7 +261,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.6
+rubygems_version: 2.6.14.1
 signing_key:
 specification_version: 4
 summary: Zendesk input plugin for Embulk