embulk-input-zendesk 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 873a74eb71d079359c4e2134e08bd0b9972477a6
-  data.tar.gz: 6a8c70921ab79299a8e10d77dd9bcb034033384c
+  metadata.gz: e0911f65a242c33edcf81953f2546825701b2a4c
+  data.tar.gz: 34b91761ad01712a8ba0bb6a9c258d9555dd5a65
 SHA512:
-  metadata.gz: baa602ab6ed0a7c33ee34fe7d200f9ea2680c1a41f5c2503ea94fc0e32f69b14571f2e45e7e85a548c06741daa63ccffa227953502add3e2503d76cbb9813f4d
-  data.tar.gz: 6aae5b2d71285613c3e89bba77244385b2854afe60c4e12104f098954f7ad82ba30cd2604fa77f0e8483391fd996cc3094ccaaf89627495b00cf459a49b57a09
+  metadata.gz: 85e63afc0356899056627249dda852e1a9fe0af9de006335b86d89c01bc53ac97d93a8974f82c27ce69041ad22a40ca271e85d77853d7378cc425198657d6224
+  data.tar.gz: 01e8c2238dd8aa341f3560ed1c8079f0d68454e3a9964137483cedaf625bbec42a4c9bb6ae1cd3fefcdaa9fe4a68896b049150968efe8c4782ac06e943034d88

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.13 - 2019-01-14
+* [enhancement] Add `dedup` option, in order to avoid OOM when importing large dataset [#48](https://github.com/treasure-data/embulk-input-zendesk/pull/48)
+
 ## 0.2.12 - 2019-01-04
 * [enhancement] Fix performance issue [#47](https://github.com/treasure-data/embulk-input-zendesk/pull/47)
 

data/README.md

@@ -31,7 +31,8 @@ Required Embulk version >= 0.8.1.
 - **start_time**: Start export from this time if present. (string, default: `null`)
 - **retry_limit**: Try to retry this times (integer, default: 5)
 - **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 4)
-- **incremental**:  If false, `start_time` in next.yml would not be updated that means you always fetch all of data from Zendesk with statically conditions. If true, `start_time` would be updated in next.yml. (bool, default: true)
+- **incremental**: If false, `start_time` in next.yml would not be updated that means you always fetch all of data from Zendesk with statically conditions. If true, `start_time` would be updated in next.yml. (bool, default: `true`)
+- **dedup**: Zendesk incremental API is not designed to protect against duplication. In order to de-dup records, plugin has to cache fetched IDs in memory. If you're importing a large dataset (eg. tens of millions of records), it can lead to OOM error, depends on your configured heap size. In such cases, you can set this option to `false`, but keep in mind that result may contain duplicated records. (bool, default: `true`)
 - **app_marketplace_integration_name**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This should be used to name of the integration.
 - **app_marketplace_org_id**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This should be the Organization ID for your organization from the new developer portal.
 - **app_marketplace_app_id**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This is the “App ID” that will be assigned to you when you submit your app.

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.2.12"
+  spec.version       = "0.2.13"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -82,23 +82,23 @@ module Embulk
         # they have both Incremental API and non-incremental API
         # 170717: `ticket_events` can use standard endpoint format now, ie. `<target>.json`
         %w(tickets ticket_events users organizations).each do |target|
-          define_method(target) do |partial = true, start_time = 0, &block|
+          define_method(target) do |partial = true, start_time = 0, dedup = true, &block|
             # Always use incremental_export. There is some difference between incremental_export and export.
-            incremental_export("/api/v2/incremental/#{target}.json", target, start_time, Set.new, partial, &block)
+            incremental_export("/api/v2/incremental/#{target}.json", target, start_time, dedup, Set.new, partial, &block)
           end
         end
 
         # Ticket metrics will need to be export using both the non incremental and incremental on ticket
         # We provide support by filter out ticket_metrics with created at smaller than start time
         # while passing the incremental start time to the incremental ticket/ticket_metrics export
-        define_method('ticket_metrics') do |partial = true, start_time = 0, &block|
+        define_method('ticket_metrics') do |partial = true, start_time = 0, dedup = true, &block|
           if partial
             # If partial export then we need to use the old end point. Since new end point return both ticket and
             # ticket metric with ticket come first so the current approach that cut off the response packet won't work
             # Since partial is only use for preview and guess so this should be fine
             export('/api/v2/ticket_metrics.json', 'ticket_metrics', &block)
           else
-            incremental_export('/api/v2/incremental/tickets.json', 'metric_sets', start_time, Set.new, partial, { include: 'metric_sets' }, &block)
+            incremental_export('/api/v2/incremental/tickets.json', 'metric_sets', start_time, dedup, Set.new, partial, { include: 'metric_sets' }, &block)
           end
         end
 
@@ -175,7 +175,7 @@ module Embulk
           end
         end
 
-        def incremental_export(path, key, start_time = 0, known_ids = Set.new, partial = true, query = {}, &block)
+        def incremental_export(path, key, start_time = 0, dedup = true, known_ids = Set.new, partial = true, query = {}, &block)
           if partial
             records = request_partial(path, query.merge(start_time: start_time)).first(5)
             records.uniq{|r| r["id"]}.each do |record|
@@ -184,6 +184,10 @@ module Embulk
             return
           end
 
+          if !dedup
+            Embulk.logger.warn("!!! You've selected to skip de-duplicating records, result may contain duplicated data !!!")
+          end
+
           execute_thread_pool do |pool|
             loop do
               start_fetching = Time.now
@@ -208,9 +212,11 @@ module Embulk
                 # de-duplicated records.
                 # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
                 # https://github.com/zendesk/zendesk_api_client_rb/issues/251
-                next if known_ids.include?(record["id"])
+                if dedup
+                  next if known_ids.include?(record["id"])
+                  known_ids << record["id"]
+                end
 
-                known_ids << record["id"]
                 pool.post { block.call record }
                 actual_fetched += 1
               end

data/lib/embulk/input/zendesk/plugin.rb

@@ -94,6 +94,7 @@ module Embulk
             retry_limit: config.param("retry_limit", :integer, default: 5),
             retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 4),
             incremental: config.param("incremental", :bool, default: true),
+            dedup: config.param("dedup", :bool, default: true),
             schema: config.param(:columns, :array, default: []),
             includes: config.param(:includes, :array, default: []),
             app_marketplace_integration_name: config.param("app_marketplace_integration_name", :string, default: nil),
@@ -109,8 +110,11 @@ module Embulk
         def run
           method = task[:target]
           args = [preview?]
-          if @start_time
-            args << @start_time.to_i
+          args << (@start_time || 0).to_i
+
+          # de-dup may lead to OOM
+          if !task[:dedup].nil? && !task[:dedup]
+            args << false
           end
 
           mutex = Mutex.new

data/test/embulk/input/zendesk/test_client.rb

@@ -196,6 +196,28 @@ module Embulk
               assert_equal(2,counter.value)
             end
 
+            test "allows to fetch tickets metrics *with* duplicated" do
+              records = [
+                {"id" => 1, "ticket_id" => 100},
+                {"id" => 2, "ticket_id" => 200},
+                {"id" => 1, "ticket_id" => 100},
+                {"id" => 1, "ticket_id" => 100},
+              ]
+              @httpclient.test_loopback_http_response << [
+                "HTTP/1.1 200",
+                "Content-Type: application/json",
+                "",
+                {
+                  metric_sets: records,
+                  count: records.length,
+                }.to_json
+              ].join("\r\n")
+              counter = Concurrent::AtomicFixnum.new(0)
+              handler = proc {counter.increment}
+              client.ticket_metrics(false, 0, false, &handler)
+              assert_equal(4,counter.value)
+            end
+
             test "fetch ticket_metrics with next_page" do
               end_time = 1488535542
               response_1 = [
@@ -259,12 +281,12 @@ module Embulk
 
           sub_test_case "ticket_events" do
             test "invoke incremental_export when partial=true" do
-              mock(client).incremental_export(anything, "ticket_events", anything, Set.new, true)
+              mock(client).incremental_export(anything, "ticket_events", anything, true, Set.new, true)
               client.ticket_events(true)
             end
 
             test "invoke incremental_export when partial=false" do
-              mock(client).incremental_export(anything, "ticket_events", anything, Set.new, false)
+              mock(client).incremental_export(anything, "ticket_events", anything, true, Set.new, false)
               client.ticket_events(false)
             end
           end

data/test/embulk/input/zendesk/test_plugin.rb

@@ -340,7 +340,7 @@ module Embulk
 
             test "call tickets method instead of ticket_all" do
               mock(@client).export.never
-              mock(@client).incremental_export(anything, "tickets", anything, anything, anything) { [] }
+              mock(@client).incremental_export(anything, "tickets", anything, anything, anything, anything) { [] }
               mock(page_builder).finish
 
               @plugin.run
@@ -379,7 +379,7 @@ module Embulk
 
             test "call ticket_all method instead of tickets" do
               mock(@client).export.never
-              mock(@client).incremental_export(anything, "tickets", 0, Set.new, false) { [] }
+              mock(@client).incremental_export(anything, "tickets", 0, true, Set.new, false) { [] }
               mock(page_builder).finish
 
               @plugin.run
@@ -544,7 +544,7 @@ module Embulk
               test "Nothing passed to client" do
                 stub(page_builder).finish
 
-                mock(@client).tickets(false)
+                mock(@client).tickets(false, 0)
                 @plugin.run
               end
             end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-07 00:00:00.000000000 Z
+date: 2019-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement