embulk-input-zendesk 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73115292227dc5474093e998c1025f4a0fa55c22
-  data.tar.gz: 23cd509f73bfa2c8ccff24478b258295c32c8806
+  metadata.gz: eaddbf9ca58a72b5bb9e00497f7eef3c3303e171
+  data.tar.gz: 4a13d21ceff5b8d019f43d9c98ea4f0dac59840c
 SHA512:
-  metadata.gz: 6364750da3c79b7dc8a8e9507887884d3cda61445ad8f775b7bffbfb764f07a49938621380060c323c43a1a806b4ffc20f1d69a3e8e045ec7c12aa0e8d1c84a3
-  data.tar.gz: 93ae137dacd61c9b56f323916f6993a6f7b077231646227e85003c85d1966ee8b66c1ab99cf87c922ccc55f021ff5edeb8f136aad5039448a6d99fc5af98961d
+  metadata.gz: fe9134f35bc6d9d0a303f103c7d85ac017eb43a2c2db662c8ae7d18bac85afaed44572cefa6c6fab21ac1c5f6a35861ee61088edcac72421b2fb018844bf3e96
+  data.tar.gz: 3933f732fd8f073fc52f9ce44962a7e9b8ce6ba248e77d9c45cf1468093ded9f21a0092cb29e7c9b7b779bf820211388f9c0365412e10f52768cfc19dc341a7e

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.8 - 2016-07-11
+
+* [enhancement] For huge data [#13](https://github.com/treasure-data/embulk-input-zendesk/pull/13)
+* [enhancement] Improvements for non incremental export [#12](https://github.com/treasure-data/embulk-input-zendesk/pull/12)
+
 ## 0.1.7 - 2016-06-04
 * [enhancement] Improvements for non incremental export [#12](https://github.com/treasure-data/embulk-input-zendesk/pull/12)
 

data/README.md

@@ -30,7 +30,7 @@ Required Embulk version >= 0.8.1.
 - **access_token**: OAuth Access Token. required if `auth_method` is `oauth`. (string, default: `null`)
 - **start_time**: Start export from this time if present. (string, default: `null`)
 - **retry_limit**: Try to retry this times (integer, default: 5)
-- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1)
+- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 4)
 - **incremental**:  If false, `start_time` in next.yml would not be updated that means you always fetch all of data from Zendesk with statically conditions. If true, `start_time` would be updated in next.yml. (bool, default: true)
 
 ## Example

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.7"
+  spec.version       = "0.1.8"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -20,6 +20,7 @@ module Embulk
 
         def httpclient
           httpclient = HTTPClient.new
+          httpclient.connect_timeout = 240 # default:60 is not enough for huge data
           # httpclient.debug_dev = STDOUT
           return set_auth(httpclient)
         end
@@ -162,35 +163,39 @@ module Embulk
         def incremental_export(path, key, start_time = 0, known_ids = [], partial = true, &block)
           if partial
             records = request_partial(path, {start_time: start_time}).first(5)
-          else
+            records.uniq{|r| r["id"]}.each do |record|
+              block.call record
+            end
+            return
+          end
+
+          loop do
+            start_fetching = Time.now
             response = request(path, {start_time: start_time})
             begin
               data = JSON.parse(response.body)
             rescue => e
               raise Embulk::DataError.new(e)
             end
-            Embulk.logger.info "Fetched records from #{start_time} (#{Time.at(start_time)})"
+            actual_fetched = 0
             records = data[key]
-          end
-
-          records.each do |record|
-            # de-duplicated records.
-            # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
-            # https://github.com/zendesk/zendesk_api_client_rb/issues/251
-            next if known_ids.include?(record["id"])
-
-            known_ids << record["id"]
-            block.call record
-          end
-          return if partial
+            records.each do |record|
+              # de-duplicated records.
+              # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
+              # https://github.com/zendesk/zendesk_api_client_rb/issues/251
+              next if known_ids.include?(record["id"])
+
+              known_ids << record["id"]
+              block.call record
+              actual_fetched += 1
+            end
+            Embulk.logger.info "Fetched #{actual_fetched} records from start_time:#{start_time} (#{Time.at(start_time)}) within #{Time.now.to_i - start_fetching.to_i} seconds"
+            start_time = data["end_time"]
 
-          # NOTE: If count is less than 1000, then stop paginating.
-          #       Otherwise, use the next_page URL to get the next page of results.
-          #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
-          if data["count"] == 1000
-            incremental_export(path, key, data["end_time"], known_ids, partial, &block)
-          else
-            data
+            # NOTE: If count is less than 1000, then stop paginating.
+            #       Otherwise, use the next_page URL to get the next page of results.
+            #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
+            break data if data["count"] < 1000
           end
         end
 

data/test/embulk/input/zendesk/test_plugin.rb

@@ -375,12 +375,14 @@ module Embulk
                 "Content-Type: application/json",
                 "",
                 {
-                  tickets: tickets
+                  tickets: tickets,
+                  count: tickets.length,
                 }.to_json
               ].join("\r\n")
 
               tickets.each do |ticket|
-                mock(page_builder).add([ticket["id"]])
+                # schema[:columns] is id and tags. tags should be nil
+                mock(page_builder).add([ticket["id"], nil])
               end
               mock(page_builder).finish
 
@@ -413,6 +415,7 @@ module Embulk
                   {
                     ticket_events: events,
                     end_time: end_time,
+                    count: events.length,
                   }.to_json
                 ].join("\r\n")
                 stub(page_builder).add(anything)
@@ -452,7 +455,8 @@ module Embulk
                   "Content-Type: application/json",
                   "",
                   {
-                    tickets: data
+                    tickets: data,
+                    count: data.length,
                   }.to_json
                 ].join("\r\n")
               end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-03 00:00:00.000000000 Z
+date: 2016-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement