embulk-input-zendesk 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ed2bcc09ae824662952c09511b06045ebef3641
-  data.tar.gz: 95281d1ac59e524c36356c9c670de71c067fb326
+  metadata.gz: dd6da90d8ce5a50d324d5d0d4e3016b151ebcdde
+  data.tar.gz: adff4881ad79e42236e68f6245cf0ab15dd16463
 SHA512:
-  metadata.gz: 4994efad0da2b8872ceda6c29a0cd5eda72136194a33e71c6e6a418dbf412f95c289463d243c7e812b30c23a56f79dee19c344461667b722e10231c44cdcdfb5
-  data.tar.gz: b9b63e1fde1f724cc418e39a529c9bf0f3ebbf831bfdd5f268ef3027b71b25fa7987f3370db058ce51c12e9e704f01a0bac55d40f26305104a3debf92fd60ab1
+  metadata.gz: fbf47f6f6f96ab00bf2642c313b3829a3a85356e339bd65cb32e26bfa847a5e57200dafc31fa1b47b76b8828bd34b94a8d9baafca0f40c285b3a31d0504d63cd
+  data.tar.gz: eb56a55ecf3d7ddb5dc2785a63e3daa7909267a0bac1c7ed52f2fce80e4103d8a74bac9d3dcc631c49f8436252e80f7dd82fed64fa4f9bb6049577594868ebe5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.4 - 2016-04-08
+
+* [enhancement] Correct preview data [#9](https://github.com/treasure-data/embulk-input-zendesk/pull/9)
+
 ## 0.1.3 - 2016-03-15
 
 * [enhancement] Support more targets [#8](https://github.com/treasure-data/embulk-input-zendesk/pull/8)

data/embulk-input-zendesk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-zendesk"
-  spec.version       = "0.1.3"
+  spec.version       = "0.1.4"
   spec.authors       = ["uu59", "muga", "sakama"]
   spec.summary       = "Zendesk input plugin for Embulk"
   spec.description   = "Loads records from Zendesk."

data/lib/embulk/input/zendesk/client.rb

@@ -1,3 +1,4 @@
+require "strscan"
 require "httpclient"
 
 module Embulk
@@ -7,6 +8,7 @@ module Embulk
         attr_reader :config
 
         PARTIAL_RECORDS_SIZE = 50
+        PARTIAL_RECORDS_BYTE_SIZE = 50000
         AVAILABLE_INCREMENTAL_EXPORT = %w(tickets users organizations ticket_events).freeze
         UNAVAILABLE_INCREMENTAL_EXPORT = %w(ticket_fields ticket_forms ticket_metrics).freeze
         AVAILABLE_TARGETS = AVAILABLE_INCREMENTAL_EXPORT + UNAVAILABLE_INCREMENTAL_EXPORT
@@ -52,11 +54,8 @@ module Embulk
         # they have both Incremental API and non-incremental API
         %w(tickets users organizations).each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
-            if partial
-              export("/api/v2/#{target}.json", target, PARTIAL_RECORDS_SIZE, &block) # Ignore start_time
-            else
-              incremental_export("/api/v2/incremental/#{target}.json", target, start_time, [], &block)
-            end
+            # Always use incremental_export. There is some difference between incremental_export and export.
+            incremental_export("/api/v2/incremental/#{target}.json", target, start_time, [], partial, &block)
           end
         end
 
@@ -64,7 +63,7 @@ module Embulk
         %w(ticket_events).each do |target|
           define_method(target) do |partial = true, start_time = 0, &block|
             path = "/api/v2/incremental/#{target}"
-            incremental_export(path, target, start_time, [], &block)
+            incremental_export(path, target, start_time, [], partial, &block)
           end
         end
 
@@ -109,18 +108,21 @@ module Embulk
           nil # this is necessary different with incremental_export
         end
 
-        def incremental_export(path, key, start_time = 0, known_ids = [], &block)
-          # for `embulk run` to fetch all records.
-          response = request(path, start_time: start_time)
-
-          begin
-            data = JSON.parse(response.body)
-          rescue => e
-            raise Embulk::DataError.new(e)
+        def incremental_export(path, key, start_time = 0, known_ids = [], partial = true, &block)
+          if partial
+            records = request_partial(path, {start_time: start_time}).first(5)
+          else
+            response = request(path, {start_time: start_time})
+            begin
+              data = JSON.parse(response.body)
+            rescue => e
+              raise Embulk::DataError.new(e)
+            end
+            Embulk.logger.debug "start_time:#{start_time} (#{Time.at(start_time)}) count:#{data["count"]} next_page:#{data["next_page"]} end_time:#{data["end_time"]} "
+            records = data[key]
           end
 
-          Embulk.logger.debug "start_time:#{start_time} (#{Time.at(start_time)}) count:#{data["count"]} next_page:#{data["next_page"]} end_time:#{data["end_time"]} "
-          data[key].each do |record|
+          records.each do |record|
             # de-duplicated records.
             # https://developer.zendesk.com/rest_api/docs/core/incremental_export#usage-notes
             # https://github.com/zendesk/zendesk_api_client_rb/issues/251
@@ -129,12 +131,13 @@ module Embulk
             known_ids << record["id"]
             block.call record
           end
+          return if partial
 
           # NOTE: If count is less than 1000, then stop paginating.
           #       Otherwise, use the next_page URL to get the next page of results.
           #       https://developer.zendesk.com/rest_api/docs/core/incremental_export#pagination
           if data["count"] == 1000
-            incremental_export(path, key, data["end_time"], known_ids, &block)
+            incremental_export(path, key, data["end_time"], known_ids, partial, &block)
           else
             data
           end
@@ -175,32 +178,66 @@ module Embulk
             Embulk.logger.debug "Fetching #{u.to_s}"
             response = httpclient.get(u.to_s, query, follow_redirect: true)
 
-            # https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
-            status_code = response.status
-            case status_code
-            when 200, 404
-              # 404 would be returned e.g. ticket comments are empty (on fetch_subresource method)
-              response
-            when 400, 401
-              raise Embulk::ConfigError.new("[#{status_code}] #{response.body}")
-            when 409
-              raise "[#{status_code}] temporally failure."
-            when 429
-              # rate limit
-              retry_after = response.headers["Retry-After"]
-              wait_rate_limit(retry_after.to_i)
-            when 500, 503
-              # 503 is possible rate limit
-              retry_after = response.headers["Retry-After"]
-              if retry_after
-                wait_rate_limit(retry_after.to_i)
-              else
-                raise "[#{status_code}] temporally failure."
+            handle_response(response.status, response.headers, response.body)
+            response
+          end
+        end
+
+        def request_partial(path, query = {})
+          # NOTE: This is a dirty hack for quick response using incremental_export API.
+          #       Disconnect socket when received PARTIAL_RECORDS_BYTE_SIZE bytes,
+          #       And extract valid JSONs from received bytes (extract_valid_json_from_chunk method)
+          u = URI.parse(config[:login_url])
+          u.path = path
+
+          retryer.with_retry do
+            Embulk.logger.debug "Fetching #{u.to_s}"
+            buf = ""
+            auth_retry = 0
+            httpclient.get(u.to_s, query, follow_redirect: true) do |message, chunk|
+              if message.status == 401
+                # First request will fail by 401 because not included credentials.
+                # HTTPClient will retry request with credentials.
+                if auth_retry.zero?
+                  auth_retry += 1
+                  next
+                end
+              end
+              handle_response(message.status, message.headers, chunk)
+
+              buf << chunk
+              break if buf.bytesize > PARTIAL_RECORDS_BYTE_SIZE
+            end
+            extract_valid_json_from_chunk(buf).map do |json|
+              JSON.parse(json)
+            end
+          end
+        end
+
+        def extract_valid_json_from_chunk(chunk)
+          # Drip JSON objects from incomplete string
+          #
+          # e.g.:
+          # chunk = '{"ticket_events":[{"foo":1},{"foo":2},{"fo'
+          # extract_valid_json_from_chunk(chunk) #=>  ['{"foo":1}', '{"foo":2}']
+          result = []
+
+          # omit '{"tickets":[' prefix. See test/fixtures/tickets.json for actual response.
+          s = StringScanner.new(chunk.scrub.gsub(%r!^{".*?":\[!,""))
+          while !s.eos?
+            opener = s.scan(/{/)
+            break unless opener
+            buf = opener # Initialize `buf` as "{"
+            while content = s.scan(/.*?}/) # grab data from start to next "}"
+              buf << content
+              if (JSON.parse(buf) rescue false) # if JSON.parse success, `buf` is valid JSON. we'll take it.
+                result << buf.dup
+                break
               end
-            else
-              raise "Server returns unknown status code (#{status_code})"
             end
+            s.scan(/[^{]*/) # skip until next "{". `chunk` has comma separeted objects like  '},{'. skip that comma.
           end
+          result
         end
 
         def wait_rate_limit(retry_after)
@@ -209,6 +246,31 @@ module Embulk
           throw :retry
         end
 
+        def handle_response(status_code, headers, body)
+          # https://developer.zendesk.com/rest_api/docs/core/introduction#response-format
+          case status_code
+          when 200, 404
+            # 404 would be returned e.g. ticket comments are empty (on fetch_subresource method)
+          when 400, 401
+            raise Embulk::ConfigError.new("[#{status_code}] #{body}")
+          when 409
+            raise "[#{status_code}] temporally failure."
+          when 429
+            # rate limit
+            retry_after = headers["Retry-After"]
+            wait_rate_limit(retry_after.to_i)
+          when 500, 503
+            # 503 is possible rate limit
+            retry_after = headers["Retry-After"]
+            if retry_after
+              wait_rate_limit(retry_after.to_i)
+            else
+              raise "[#{status_code}] temporally failure."
+            end
+          else
+            raise "Server returns unknown status code (#{status_code})"
+          end
+        end
       end
     end
   end

data/lib/embulk/input/zendesk/plugin.rb

@@ -96,7 +96,7 @@ module Embulk
         def run
           method = task[:target]
           args = [preview?]
-          if !preview? && @start_time
+          if @start_time
             args << @start_time.to_i
           end
 
@@ -104,6 +104,7 @@ module Embulk
             record = fetch_related_object(record)
             values = extract_values(record)
             page_builder.add(values)
+            break if preview? # NOTE: preview take care only 1 record. subresources fetching is slow.
           end
           page_builder.finish
 
@@ -123,12 +124,7 @@ module Embulk
 
         def fetch_related_object(record)
           (task[:includes] || []).each do |ent|
-            if preview?
-              # Fetching subresource consume ~2 sec for each record. it is too long to preview. so the dummy value used.
-              record[ent] = [{dummy: "(#{ent}) dummy value for preview"}]
-            else
-              record[ent] = client.fetch_subresource(record["id"], task[:target], ent)
-            end
+            record[ent] = client.fetch_subresource(record["id"], task[:target], ent)
           end
           record
         end

data/test/embulk/input/zendesk/test_client.rb

@@ -47,19 +47,6 @@ module Embulk
               end
               client.tickets(&handler)
             end
-
-            test "raise DataError when invalid JSON response" do
-              @httpclient.test_loopback_http_response << [
-                "HTTP/1.1 200",
-                "Content-Type: application/json",
-                "",
-                "[[[" # invalid json
-              ].join("\r\n")
-
-              assert_raise(DataError) do
-                client.tickets
-              end
-            end
           end
 
           sub_test_case "all" do
@@ -175,12 +162,12 @@ module Embulk
 
           sub_test_case "ticket_events" do
             test "invoke incremental_export when partial=true" do
-              mock(client).incremental_export(anything, "ticket_events", anything, [])
+              mock(client).incremental_export(anything, "ticket_events", anything, [], true)
               client.ticket_events(true)
             end
 
             test "invoke incremental_export when partial=false" do
-              mock(client).incremental_export(anything, "ticket_events", anything, [])
+              mock(client).incremental_export(anything, "ticket_events", anything, [], false)
               client.ticket_events(false)
             end
           end
@@ -364,13 +351,6 @@ module Embulk
             end
           end
 
-          test "401" do
-            stub_response(401)
-            assert_raise(ConfigError) do
-              client.tickets(&proc{})
-            end
-          end
-
           test "409" do
             stub_response(409)
             assert_raise(StandardError) do
@@ -444,6 +424,27 @@ module Embulk
           end
         end
 
+        sub_test_case ".extract_valid_json_from_chunk" do
+          setup do
+            @client = Client.new({target: "tickets"})
+          end
+
+          test "complete json" do
+            actual = @client.send(:extract_valid_json_from_chunk, '{"tickets":[{"foo":1},{"foo":2}]}')
+            assert_equal ['{"foo":1}', '{"foo":2}'], actual
+          end
+
+          test "broken json" do
+            json = '{"ticket_events":[{"foo":1},{"foo":2},{"fo'
+            actual = @client.send(:extract_valid_json_from_chunk, json)
+            expected = [
+              '{"foo":1}',
+              '{"foo":2}',
+            ]
+            assert_equal expected, actual
+          end
+        end
+
         def login_url
           "http://example.com"
         end

data/test/embulk/input/zendesk/test_plugin.rb

@@ -106,11 +106,7 @@ module Embulk
               "HTTP/1.1 200",
               "Content-Type: application/json",
               "",
-              {
-                tickets: [
-                  JSON.parse(fixture_load("tickets.json"))
-                ]
-              }.to_json
+              JSON.parse(fixture_load("tickets.json")).to_json
             ].join("\r\n")
             mock(@client).validate_config
             Plugin.guess(config)["columns"]
@@ -121,11 +117,7 @@ module Embulk
               "HTTP/1.1 200",
               "Content-Type: application/json",
               "",
-              {
-                tickets: [
-                  JSON.parse(fixture_load("tickets.json"))
-                ]
-              }.to_json
+              JSON.parse(fixture_load("tickets.json")).to_json
             ].join("\r\n")
             actual = Plugin.guess(config)["columns"]
             assert actual.include?(name: "url", type: :string)
@@ -241,7 +233,7 @@ module Embulk
 
                 test "call fetch_subresource" do
                   includes.each do |ent|
-                    mock(@client).fetch_subresource(anything, anything, ent).never
+                    mock(@client).fetch_subresource(anything, anything, ent)
                   end
                   @plugin.run
                 end
@@ -326,8 +318,8 @@ module Embulk
             end
 
             test "call tickets method instead of ticket_all" do
-              mock(@client).export(anything, "tickets", anything) { [] }
-              mock(@client).incremental_export.never
+              mock(@client).export.never
+              mock(@client).incremental_export(anything, "tickets", anything, anything, anything) { [] }
               mock(page_builder).finish
 
               @plugin.run
@@ -348,9 +340,10 @@ module Embulk
                 }.to_json
               ].join("\r\n")
 
-              tickets.each do |ticket|
-                mock(page_builder).add([ticket["id"], ticket["tags"]])
-              end
+              first_ticket = tickets[0]
+              second_ticket = tickets[1]
+              mock(page_builder).add([first_ticket["id"], first_ticket["tags"]])
+              mock(page_builder).add([second_ticket["id"], second_ticket["tags"]]).never
               mock(page_builder).finish
 
               @plugin.run
@@ -364,7 +357,7 @@ module Embulk
 
             test "call ticket_all method instead of tickets" do
               mock(@client).export.never
-              mock(@client).incremental_export(anything, "tickets", 0, []) { [] }
+              mock(@client).incremental_export(anything, "tickets", 0, [], false) { [] }
               mock(page_builder).finish
 
               @plugin.run

data/test/fixtures/tickets.json

@@ -1,4 +1,4 @@
-{
+{"tickets":[{
   "id":               35436,
   "url":              "https://company.zendesk.com/api/v2/tickets/35436.json",
   "external_id":      "ahg35h3jh",
@@ -6,7 +6,7 @@
   "updated_at":       "2011-05-05T10:38:52Z",
   "type":             "incident",
   "subject":          "Help, my printer is on fire!",
-  "raw_subject":      "{{dc.printer_on_fire}}",
+  "raw_subject":      "{{dc.printer_on_fire}} }{",
   "description":      "The fire is very colorful.",
   "priority":         "high",
   "status":           "open",
@@ -41,4 +41,4 @@
     "comment": "Great support!"
   },
   "sharing_agreement_ids": [84432]
-}
+}]}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-15 00:00:00.000000000 Z
+date: 2016-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement