embulk-input-zendesk 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c75172d0513587f42bb1e6b14128aa846df1058e
-  data.tar.gz: 586ec6ed68c05bdfca67fd33d8def900084b0465
+  metadata.gz: eddb031e2e41652c6c825d2caaefaf618125fa07
+  data.tar.gz: 2b76a1a7a750008b20823c8dbb7f4794ad3d92dd
 SHA512:
-  metadata.gz: 8a3bfcf0328d070f5030171760fdc1c76643d960287cd0ed441f8fda1f775fe32bc4f4fc192106893544395a0b4f69beec3672d7a1ca4ffe9ae5338cc04f27ee
-  data.tar.gz: 407f04e70f84022c34708b20585e5dcf4e9a158acf4a200fb3a9d1ef583b76c3e11b15da736772f18b8619a237e9360fd5267b18d488708ce2463399fd94e586
+  metadata.gz: eb4d2dc9a0b128bf2c66f259750b9db6f27f9d67b512e48a94cd1d4c4dfd1aa52b8676c37a9f5449d43f6118605bb820d3b06e5f7b5b2020af147c172d85881f
+  data.tar.gz: 519e20eebcb4a732174b6b71be9ae01f097b00108b2d47e71299a4adc6b205fbf34a9c160beebdf31acf7609e2ac9103b6b615a2776838e4658aa1fb1bbb41b3

data/CHANGELOG.md

@@ -1,6 +1,11 @@
+## 0.1.2 - 2016-01-29
+
+* [maintenance] Add authors @muga and @sakama.
+* [enhancement] Add Incremental option [#6](https://github.com/treasure-data/embulk-input-zendesk/pull/6)
+
 ## 0.1.1 - 2016-01-26
 
-- [fixed] Fix when null value given.
+* [fixed] Fix when null value given.
 
 ## 0.1.0 - 2016-01-26
 

data/README.md

@@ -14,7 +14,7 @@ Required Embulk version >= 0.8.1.
 **NOTE** This plugin don't support JSON type columns e.g. custom fields, tags, etc for now. But they will be supported soon.
 
 * **Plugin type**: input
-* **Resume supported**: no
+* **Resume supported**: yes
 * **Cleanup supported**: no
 * **Guess supported**: yes
 
@@ -30,6 +30,7 @@ Required Embulk version >= 0.8.1.
 - **start_time**: Start export from this time if present. (string, default: `null`)
 - **retry_limit**: Try to retry this times (integer, default: 5)
 - **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1)
+- **incremental**:  If false, `start_time` in next.yml would not be updated that means you always fetch all of data from Zendesk with statically conditions. If true, `start_time` would be updated in next.yml. (bool, default: true)
 
 ## Example
 

data/embulk-input-zendesk.gemspec

@@ -1,13 +1,13 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.1"
-  spec.authors       = ["uu59"]
+  spec.version       = "0.1.2"
+  spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."
-  spec.email         = ["k@uu59.org"]
+  spec.email         = ["k@uu59.org", "muga.nishizawa@gmail.com", "satoshiakama@gmail.com"]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/k/embulk-input-zendesk"
+  spec.homepage      = "https://github.com/treasure-data/embulk-input-zendesk"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})

data/lib/embulk/input/zendesk/client.rb

@@ -11,6 +11,7 @@ module Embulk
           tickets ticket_events users organizations
           ticket_fields ticket_forms
         ).freeze
+        AVAILABLE_INCREMENTAL_EXPORT = AVAILABLE_TARGETS - %w(ticket_fields ticket_forms)
 
         def initialize(config)
           @config = config
@@ -95,6 +96,8 @@ module Embulk
           data[key].each do |record|
             block.call record
           end
+
+          nil # this is necessary different with incremental_export
         end
 
         def incremental_export(path, key, start_time = 0, known_ids = [], &block)
@@ -123,6 +126,8 @@ module Embulk
           #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
           if data["count"] == 1000
             incremental_export(path, key, data["end_time"], known_ids, &block)
+          else
+            data
           end
         end
 

data/lib/embulk/input/zendesk/plugin.rb

@@ -18,13 +18,21 @@ module Embulk
             Column.new(nil, name, type, column["format"])
           end
 
+          if task[:incremental] && !Client::AVAILABLE_INCREMENTAL_EXPORT.include?(task[:target])
+            Embulk.logger.warn "target: #{task[:target]} don't support incremental export API. Will be ignored start_time option"
+          end
+
           resume(task, columns, 1, &control)
         end
 
         def self.resume(task, columns, count, &control)
           task_reports = yield(task, columns, count)
+          report = task_reports.first
 
           next_config_diff = {}
+          if report[:start_time]
+            next_config_diff[:start_time] = report[:start_time]
+          end
           return next_config_diff
         end
 
@@ -71,6 +79,7 @@ module Embulk
             start_time: config.param("start_time", :string, default: nil),
             retry_limit: config.param("retry_limit", :integer, default: 5),
             retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
+            incremental: config.param("incremental", :bool, default: true),
             schema: config.param(:columns, :array, default: []),
           }
         end
@@ -87,14 +96,21 @@ module Embulk
           end
 
           client = Client.new(task)
-          client.public_send(method, *args) do |record|
+          last_data = client.public_send(method, *args) do |record|
             values = extract_values(record)
             page_builder.add(values)
           end
-
           page_builder.finish
 
           task_report = {}
+          if task[:incremental] && last_data && last_data["end_time"]
+            # NOTE: start_time compared as "=>", not ">".
+            #       If we will use end_time for next start_time, we got the same record that is last fetched
+            #       end_time + 1 is workaround for that
+            next_start_time = Time.at(last_data["end_time"] + 1)
+            task_report[:start_time] = next_start_time.strftime("%Y-%m-%d %H:%M:%S%z")
+          end
+
           return task_report
         end
 

data/test/embulk/input/zendesk/test_plugin.rb

@@ -242,6 +242,61 @@ module Embulk
               @plugin.run
             end
 
+            sub_test_case "config diff" do
+              def end_time
+                1234567890
+              end
+
+              def next_start_time
+                Time.at(end_time + 1).strftime("%F %T%z")
+              end
+
+              def start_time
+                Time.at(1111111111).strftime("%F %T%z")
+              end
+
+              setup do
+                events = [
+                  {"id" => 1, "created_at" => "2000-01-01T00:00:00+0900"},
+                  {"id" => 2, "created_at" => "2000-01-01T01:00:00+0900"},
+                ]
+
+                @httpclient.test_loopback_http_response << [
+                  "HTTP/1.1 200",
+                  "Content-Type: application/json",
+                  "",
+                  {
+                    ticket_events: events,
+                    end_time: end_time,
+                  }.to_json
+                ].join("\r\n")
+                stub(page_builder).add(anything)
+                stub(page_builder).finish
+              end
+
+              sub_test_case "incremental: true" do
+                def run_task
+                  task.merge(schema: schema, target: "ticket_events", incremental: true, start_time: start_time)
+                end
+
+                test "task_report contains next start_time" do
+                  report = @plugin.run
+                  assert_equal next_start_time, report[:start_time]
+                end
+              end
+
+              sub_test_case "incremental: false" do
+                def run_task
+                  task.merge(schema: schema, target: "ticket_events", incremental: false, start_time: start_time)
+                end
+
+                test "task_report don't contains start_time" do
+                  report = @plugin.run
+                  assert_nil report[:start_time]
+                end
+              end
+            end
+
             sub_test_case "casting value" do
               setup do
                 stub(@plugin).preview? { false }

metadata

@@ -1,14 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - uu59
+- muga
+- sakama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -181,6 +183,8 @@ dependencies:
 description: Loads records from Zendesk.
 email:
 - k@uu59.org
+- muga.nishizawa@gmail.com
+- satoshiakama@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -214,7 +218,7 @@ files:
 - test/fixtures/valid_auth_token.yml
 - test/override_assert_raise.rb
 - test/run-test.rb
-homepage:
+homepage: https://github.com/treasure-data/embulk-input-zendesk
 licenses:
 - MIT
 metadata: {}