embulk-input-zendesk 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eddb031e2e41652c6c825d2caaefaf618125fa07
-  data.tar.gz: 2b76a1a7a750008b20823c8dbb7f4794ad3d92dd
+  metadata.gz: 6ed2bcc09ae824662952c09511b06045ebef3641
+  data.tar.gz: 95281d1ac59e524c36356c9c670de71c067fb326
 SHA512:
-  metadata.gz: eb4d2dc9a0b128bf2c66f259750b9db6f27f9d67b512e48a94cd1d4c4dfd1aa52b8676c37a9f5449d43f6118605bb820d3b06e5f7b5b2020af147c172d85881f
-  data.tar.gz: 519e20eebcb4a732174b6b71be9ae01f097b00108b2d47e71299a4adc6b205fbf34a9c160beebdf31acf7609e2ac9103b6b615a2776838e4658aa1fb1bbb41b3
+  metadata.gz: 4994efad0da2b8872ceda6c29a0cd5eda72136194a33e71c6e6a418dbf412f95c289463d243c7e812b30c23a56f79dee19c344461667b722e10231c44cdcdfb5
+  data.tar.gz: b9b63e1fde1f724cc418e39a529c9bf0f3ebbf831bfdd5f268ef3027b71b25fa7987f3370db058ce51c12e9e704f01a0bac55d40f26305104a3debf92fd60ab1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.3 - 2016-03-15
+
+* [enhancement] Support more targets [#8](https://github.com/treasure-data/embulk-input-zendesk/pull/8)
+* [enhancement] Enable json type [#7](https://github.com/treasure-data/embulk-input-zendesk/pull/7)
+
 ## 0.1.2 - 2016-01-29
 
 * [maintenance] Add authors @muga and @sakama.

data/README.md

@@ -22,7 +22,8 @@ Required Embulk version >= 0.8.1.
 
 - **login_url**: Login URL for Zendesk (string, required)
 - **auth_method**: `basic`, `token`, or `oauth`. For more detail on [zendesk document](https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication). (string, required)
-- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields` or `ticket_forms`. (string, required)
+- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields`, `ticket_forms` or `ticket_metrics`. (string, required)
+- **includes**: Will fetch sub resources. For example, ticket has ticket_audits, ticket_comments. See below example config. (array, default: `[]`)
 - **username**: The user name a.k.a. email. Required if `auth_method` is `basic` or `token`. (string, default: `null`)
 - **password**: Password. required if `auth_method` is `basic`. (string, default: `null`)
 - **token**: Token. required if `auth_method` is `token`. (string, default: `null`)
@@ -42,6 +43,9 @@ in:
   username: jdoe@example.com
   token: 6wiIBWbGkBMo1mRDMuVwkw1EPsNkeUj95PIz2akv
   target: tickets
+  includes:
+    - audits
+    - comments
   start_time: "2015-01-01 00:00:00+0000"
 ```
 

data/Rakefile

@@ -5,7 +5,7 @@ task default: :test
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
 end
 
 desc "Run tests with coverage"

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -7,11 +7,9 @@ module Embulk
         attr_reader :config
 
         PARTIAL_RECORDS_SIZE = 50
-        AVAILABLE_TARGETS = %w(
-          tickets ticket_events users organizations
-          ticket_fields ticket_forms
-        ).freeze
-        AVAILABLE_INCREMENTAL_EXPORT = AVAILABLE_TARGETS - %w(ticket_fields ticket_forms)
+        AVAILABLE_INCREMENTAL_EXPORT = %w(tickets users organizations ticket_events).freeze
+        UNAVAILABLE_INCREMENTAL_EXPORT = %w(ticket_fields ticket_forms ticket_metrics).freeze
+        AVAILABLE_TARGETS = AVAILABLE_INCREMENTAL_EXPORT + UNAVAILABLE_INCREMENTAL_EXPORT
 
         def initialize(config)
           @config = config
@@ -47,10 +45,11 @@ module Embulk
 
         def validate_target
           unless AVAILABLE_TARGETS.include?(config[:target])
-            raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported.")
+            raise Embulk::ConfigError.new("target: '#{config[:target]}' is not supported. Supported targets are #{AVAILABLE_TARGETS.join(", ")}.")
           end
         end
 
+        # they have both Incremental API and non-incremental API
         %w(tickets users organizations).each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
             if partial
@@ -61,22 +60,32 @@ module Embulk
           end
         end
 
-        def ticket_events(partial = true, start_time = 0, &block)
-          # NOTE: ticket_events only have incremental export API
-          path = "/api/v2/incremental/ticket_events"
-          incremental_export(path, "ticket_events", start_time, [], &block)
+        # they have incremental API only
+        %w(ticket_events).each do |target|
+          define_method(target) do |partial = true, start_time = 0, &block|
+            path = "/api/v2/incremental/#{target}"
+            incremental_export(path, target, start_time, [], &block)
+          end
         end
 
-        def ticket_fields(partial = true, start_time = 0, &block)
-          # NOTE: ticket_fields only have export API (not incremental)
-          path = "/api/v2/ticket_fields.json"
-          export(path, "ticket_fields", 1000, &block)
+        # they have non-incremental API only
+        UNAVAILABLE_INCREMENTAL_EXPORT.each do |target|
+          define_method(target) do |partial = true, start_time = 0, &block|
+            path = "/api/v2/#{target}.json"
+            export(path, target, partial ? PARTIAL_RECORDS_SIZE : 1000, &block)
+          end
         end
 
-        def ticket_forms(partial = true, start_time = 0, &block)
-          # NOTE: ticket_forms only have export API (not incremental)
-          path = "/api/v2/ticket_forms.json"
-          export(path, "ticket_forms", 1000, &block)
+        def fetch_subresource(record_id, base, target)
+          response = request("/api/v2/#{base}/#{record_id}/#{target}.json")
+          return [] if response.status == 404
+
+          begin
+            data = JSON.parse(response.body)
+            data[target]
+          rescue => e
+            raise Embulk::DataError.new(e)
+          end
         end
 
         private
@@ -163,12 +172,14 @@ module Embulk
           u.path = path
 
           retryer.with_retry do
+            Embulk.logger.debug "Fetching #{u.to_s}"
             response = httpclient.get(u.to_s, query, follow_redirect: true)
 
             # https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
             status_code = response.status
             case status_code
-            when 200
+            when 200, 404
+              # 404 would be returned e.g. ticket comments are empty (on fetch_subresource method)
               response
             when 400, 401
               raise Embulk::ConfigError.new("[#{status_code}] #{response.body}")

data/lib/embulk/input/zendesk/plugin.rb

@@ -58,12 +58,16 @@ module Embulk
               hash[:type] = :json
             end
 
-            # NOTE: current version don't support JSON type
-            next if hash[:type] == :json
-
             hash
           end
 
+          task[:includes].each do |ent|
+            columns << {
+              name: ent,
+              type: :json
+            }
+          end
+
           return {"columns" => columns.compact}
         end
 
@@ -81,6 +85,7 @@ module Embulk
             retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
             incremental: config.param("incremental", :bool, default: true),
             schema: config.param(:columns, :array, default: []),
+            includes: config.param(:includes, :array, default: []),
           }
         end
 
@@ -95,8 +100,8 @@ module Embulk
             args << @start_time.to_i
           end
 
-          client = Client.new(task)
           last_data = client.public_send(method, *args) do |record|
+            record = fetch_related_object(record)
             values = extract_values(record)
             page_builder.add(values)
           end
@@ -116,6 +121,22 @@ module Embulk
 
         private
 
+        def fetch_related_object(record)
+          (task[:includes] || []).each do |ent|
+            if preview?
+              # Fetching subresource consume ~2 sec for each record. it is too long to preview. so the dummy value used.
+              record[ent] = [{dummy: "(#{ent}) dummy value for preview"}]
+            else
+              record[ent] = client.fetch_subresource(record["id"], task[:target], ent)
+            end
+          end
+          record
+        end
+
+        def client
+          Client.new(task)
+        end
+
         def preview?
           org.embulk.spi.Exec.isPreview()
         rescue java.lang.NullPointerException => e

data/test/embulk/input/zendesk/test_plugin.rb

@@ -132,13 +132,162 @@ module Embulk
             assert actual.include?(name: "id", type: :long)
             assert actual.include?(name: "created_at", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z")
             assert actual.include?(name: "has_incidents", type: :boolean)
+            assert actual.include?(name: "tags", type: :json)
+            assert actual.include?(name: "collaborator_ids", type: :json)
+            assert actual.include?(name: "custom_fields", type: :json)
+            assert actual.include?(name: "satisfaction_rating", type: :json)
+          end
+        end
+
+        sub_test_case "include subresources" do
+          def page_builder
+            @page_builder ||= Class.new do
+              def add(_); end
+              def finish; end
+            end.new
+          end
+
+          sub_test_case "guess" do
+            def task
+              t = {
+                type: "zendesk",
+                login_url: "https://example.zendesk.com/",
+                auth_method: "token",
+                username: "foo@example.com",
+                token: "token",
+                target: "tickets",
+                includes: includes,
+              }
+              t.delete :includes unless includes
+              t
+            end
+
+            def config
+              Embulk::DataSource.new(task)
+            end
+
+            def ticket
+              JSON.parse(fixture_load("tickets.json"))
+            end
+
+            setup do
+              @client = Client.new(task)
+              stub(Client).new { @client }
+              stub(@client).public_send(anything) do |*args|
+                args.last.call(ticket)
+              end
+            end
+
+            sub_test_case "includes present" do
+              def includes
+                %w(audits comments)
+              end
+
+              test "guessed includes fields" do
+                actual = Plugin.guess(config)["columns"]
+                assert actual.include?(name: "audits", type: :json)
+                assert actual.include?(name: "comments", type: :json)
+              end
+            end
+
+            sub_test_case "includes blank" do
+              def includes
+                nil
+              end
+
+              test "not guessed includes fields" do
+                actual = Plugin.guess(config)["columns"]
+                assert !actual.include?(name: "audits", type: :json)
+                assert !actual.include?(name: "comments", type: :json)
+              end
+            end
+          end
+
+          sub_test_case "#run" do
+            def schema
+              [
+                {"name" => "id", "type" => "long"},
+                {"name" => "tags", "type" => "json"},
+              ]
+            end
+
+            def run_task
+              task.merge({
+                schema: schema,
+                retry_limit: 1,
+                retry_initial_wait_sec: 0,
+                includes: includes,
+              })
+            end
+
+            setup do
+              @client = Client.new(run_task)
+              stub(@client).public_send {|*args| args.last.call({}) }
+              @plugin = Plugin.new(run_task, nil, nil, page_builder)
+              stub(@plugin).client { @client }
+              @httpclient = @client.httpclient
+              stub(@client).httpclient { @httpclient }
+            end
+
+            sub_test_case "preview" do
+              setup do
+                stub(@plugin).preview? { true }
+              end
+
+              sub_test_case "includes present" do
+                def includes
+                  %w(foo bar)
+                end
+
+                test "call fetch_subresource" do
+                  includes.each do |ent|
+                    mock(@client).fetch_subresource(anything, anything, ent).never
+                  end
+                  @plugin.run
+                end
+              end
+
+              sub_test_case "includes blank" do
+                def includes
+                  []
+                end
+
+                test "don't call fetch_subresource" do
+                  mock(@client).fetch_subresource.never
+                  @plugin.run
+                end
+              end
+            end
 
-            # TODO: re-enable these json type tests after this plugin officially support it
-            # assert actual.include?(name: "tags", type: :json)
-            # assert actual.include?(name: "collaborator_ids", type: :json)
+            sub_test_case "run" do
+              setup do
+                stub(@plugin).preview? { false }
+              end
+
+              sub_test_case "includes present " do
+                def includes
+                  %w(foo bar)
+                end
+
+                test "call fetch_subresource" do
+                  includes.each do |ent|
+                    mock(@client).fetch_subresource(anything, anything, ent).at_least(1)
+                  end
+                  @plugin.run
+                end
+              end
 
-            # assert actual.include?(name: "custom_fields", type: :json)
-            # assert actual.include?(name: "satisfaction_rating", type: :json)
+              sub_test_case "includes blank" do
+                def includes
+                  []
+                end
+
+                test "don't call fetch_subresource" do
+                  mock(@client).fetch_subresource.never
+                  @plugin.run
+                end
+              end
+            end
           end
         end
 
@@ -149,7 +298,8 @@ module Embulk
 
           def schema
             [
-              {"name" => "id", "type" => "long"}
+              {"name" => "id", "type" => "long"},
+              {"name" => "tags", "type" => "json"},
             ]
           end
 
@@ -158,6 +308,7 @@ module Embulk
               schema: schema,
               retry_limit: 1,
               retry_initial_wait_sec: 0,
+              includes: [],
             })
           end
 
@@ -184,8 +335,8 @@ module Embulk
 
             test "task[:schema] columns passed into page_builder.add" do
               tickets = [
-                {"id" => 1, "created_at" => "2000-01-01T00:00:00+0900"},
-                {"id" => 2, "created_at" => "2000-01-01T00:00:00+0900"},
+                {"id" => 1, "created_at" => "2000-01-01T00:00:00+0900", "tags" => ["foo"]},
+                {"id" => 2, "created_at" => "2000-01-01T00:00:00+0900", "tags" => ["foo"]},
               ]
 
               @httpclient.test_loopback_http_response << [
@@ -198,7 +349,7 @@ module Embulk
               ].join("\r\n")
 
               tickets.each do |ticket|
-                mock(page_builder).add([ticket["id"]])
+                mock(page_builder).add([ticket["id"], ticket["tags"]])
               end
               mock(page_builder).finish
 
@@ -317,6 +468,7 @@ module Embulk
                   {"name" => "target_str", "type" => "string"},
                   {"name" => "target_bool", "type" => "boolean"},
                   {"name" => "target_time", "type" => "timestamp"},
+                  {"name" => "target_json", "type" => "json"},
                 ]
               end
 
@@ -325,22 +477,25 @@ module Embulk
                   {
                     "id" => 1, "target_l" => "3", "target_f" => "3", "target_str" => "str",
                     "target_bool" => false, "target_time" => "2000-01-01",
+                    "target_json" => [1,2,3],
                   },
                   {
                     "id" => 2, "target_l" => 4.5, "target_f" => 4.5, "target_str" => 999,
                     "target_bool" => "truthy", "target_time" => Time.parse("1999-01-01"),
+                    "target_json" => {"foo" => "bar"},
                   },
                   {
                     "id" => 3, "target_l" => nil, "target_f" => nil, "target_str" => nil,
                     "target_bool" => nil, "target_time" => nil,
+                    "target_json" => nil,
                   },
                 ]
               end
 
               test "cast as given type" do
-                mock(page_builder).add([3, 3.0, "str", false, Time.parse("2000-01-01")])
-                mock(page_builder).add([4, 4.5, "999", true, Time.parse("1999-01-01")])
-                mock(page_builder).add([nil, nil, nil, nil, nil])
+                mock(page_builder).add([3, 3.0, "str", false, Time.parse("2000-01-01"), [1,2,3]])
+                mock(page_builder).add([4, 4.5, "999", true, Time.parse("1999-01-01"), {"foo" => "bar"}])
+                mock(page_builder).add([nil, nil, nil, nil, nil, nil])
                 mock(page_builder).finish
 
                 @plugin.run

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement