embulk-input-marketo 0.4.0 → 0.5.0

data/test/embulk/input/marketo/test_lead.rb

@@ -126,11 +126,11 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_multiple_leads, message: request) do
-                leads_response
+                savon_response(xml_lead_response)
               end
 
               mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
-                next_stream_leads_response
+                savon_response(xml_lead_next)
               end
             end
 
@@ -140,6 +140,7 @@ module Embulk
             mock(@page_builder).add(["ten-thousand-leaf", from])
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -152,11 +153,11 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_multiple_leads, message: request) do
-                leads_response
+                savon_response(xml_lead_response)
               end
 
               mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
-                next_stream_leads_response
+                savon_response(xml_lead_next)
               end
             end
 
@@ -165,6 +166,7 @@ module Embulk
             mock(@page_builder).add(["ten-thousand-leaf"])
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -184,7 +186,7 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_multiple_leads, message: request.merge(batch_size: Lead::PREVIEW_COUNT)) do
-                preview_leads_response
+                savon_response(xml_lead_preview)
               end
             end
 
@@ -194,6 +196,7 @@ module Embulk
             end
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -204,13 +207,14 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_multiple_leads, anything) do
-                preview_leads_response
+                savon_response(xml_lead_preview)
               end
             end
 
             mock(@page_builder).add(anything).times(Lead::PREVIEW_COUNT)
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -223,25 +227,25 @@ module Embulk
               end
             end
 
-            any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
-              task[:retry_limit].times do |n|
-                mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
-              end
-            end
-
             mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
             stub(Embulk.logger).info {}
 
             assert_raise do
+              @plugin.init
               @plugin.run
             end
           end
 
           class SavonCallTest < self
             def test_soap_error
+              nori_options = {
+                :strip_namespaces          => true,
+                :convert_tags_to  => lambda { |tag| tag.snakecase.to_sym},
+                :convert_attributes_to     => lambda { |k,v| [k,v] },
+              }
               assert_raise(Embulk::ConfigError) do
                 @soap.send(:catch_unretryable_error) do
-                  raise Savon::SOAPFault.new(nil, Nori.new(default_nori_options), xml)
+                  raise Savon::SOAPFault.new(nil, Nori.new(nori_options), xml)
                 end
               end
             end
@@ -268,6 +272,7 @@ module Embulk
               stub(@soap).endpoint { "http://foo.test/" }
 
               assert_raise(Embulk::ConfigError) do
+                @plugin.init
                 @plugin.run
               end
             end
@@ -287,14 +292,6 @@ module Embulk
               XML
             end
 
-            def default_nori_options
-              # https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
-              {
-                :strip_namespaces          => true,
-                :convert_tags_to  => lambda { |tag| tag.snakecase.to_sym},
-                :convert_attributes_to     => lambda { |k,v| [k,v] },
-              }
-            end
           end
 
           class TestTimeslice < self
@@ -455,7 +452,7 @@ module Embulk
             encryption_key: "TOPSECRET",
             from_datetime: from_datetime,
             to_datetime: to_datetime,
-            retry_initial_wait_sec: 2,
+            retry_initial_wait_sec: 0,
             retry_limit: 3,
             append_processed_time_column: true,
             ranges: timeslice,

data/test/embulk/input/marketo_api/soap/test_activity_log.rb

@@ -22,18 +22,43 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_lead_changes, message: request) do
-                next_stream_activity_logs_response
+                savon_response xml_ac_next_response
               end
             end
 
             proc = proc{ "" }
-            activity_logs = next_stream_activity_logs_response[:body][:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
-            last_activity_log = activity_logs.last
+            activity_logs = savon_response(xml_ac_next_response).xpath('//leadChangeRecord')
+            latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
 
             mock(proc).call(anything).times(activity_logs.size)
-            assert_equal(last_activity_log[:activity_date_time], soap.each(from_datetime, &proc))
+            assert_equal(latest, soap.each(from_datetime, &proc))
           end
 
+          def test_no_attributes
+            request = {
+              start_position: {
+                oldest_created_at: Time.parse(from_datetime).iso8601,
+              },
+              batch_size: 100
+            }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: request) do
+                savon_response xml_ac_response_no_attributes
+              end
+            end
+
+            proc = proc{ "" }
+            activity_logs = savon_response(xml_ac_response_no_attributes).xpath('//leadChangeRecord')
+            latest = Time.parse(activity_logs.last.at('./activityDateTime').text)
+
+            mock(proc).call(anything).times(activity_logs.size)
+            assert_nothing_raised do
+              assert_equal(latest, soap.each(from_datetime, &proc))
+            end
+          end
+
+
           def test_each_with_no_response
             request = {
               start_position: {
@@ -44,7 +69,7 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_lead_changes, message: request) do
-                none_activity_log_response
+                savon_response xml_ac_none_response
               end
             end
 
@@ -63,14 +88,14 @@ module Embulk
 
             def test_savon_call
               mock(@savon).call(:get_lead_changes, message: request) {
-                next_stream_activity_logs_response
+                savon_response xml_ac_next_response
               }
               soap.metadata(from_datetime)
             end
 
             def test_return_schema
               stub(@savon).call(:get_lead_changes, message: request) {
-                next_stream_activity_logs_response
+                savon_response xml_ac_next_response
               }
               assert_equal(schema, soap.metadata(from_datetime))
             end
@@ -90,7 +115,7 @@ module Embulk
             def schema
               metadata = [
                 {index: 0, name: "id", type: :long},
-                {index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z"}, # NOTE: `format` is the same as-is response (e.g. "2015-07-14T00:00:11+0000" to "%Y-%m-%dT%H:%M:%S%z")
+                {index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%:z"}, # NOTE: `format` is the same as-is response (e.g. "2015-07-14T00:00:11+0000" to "%Y-%m-%dT%H:%M:%S%:z")
                 {index: 2, name: "activity_type", type: :string},
                 {index: 3, name: "mktg_asset_name", type: :string},
                 {index: 4, name: "mkt_person_id", type: :long},

data/test/embulk/input/marketo_api/soap/test_base.rb

@@ -20,15 +20,9 @@ module Embulk
                 stub(klass).call(:timeout_test, advanced_typecasting: false) { raise ::Timeout::Error }
               end
 
-              any_instance_of(MarketoApi::Soap::Base) do |klass|
-                retry_options[:retry_limit].times do |n|
-                  mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
-                end
-              end
-
               mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
 
-              assert_raise(::Timeout::Error) do
+              assert_raise(PerfectRetry::TooManyRetry) do
                 soap.send(:savon_call, :timeout_test, {}, retry_options)
               end
             end
@@ -38,12 +32,6 @@ module Embulk
                 stub(klass).call(:timeout_test, advanced_typecasting: false) { raise "something error" }
               end
 
-              any_instance_of(MarketoApi::Soap::Base) do |klass|
-                retry_options[:retry_limit].times do |n|
-                  mock(klass).sleep(retry_options[:retry_initial_wait_sec] * (2**n))
-                end
-              end
-
               mock(Embulk.logger).warn(/Retrying/).times(retry_options[:retry_limit])
 
               assert_raise do
@@ -68,7 +56,7 @@ module Embulk
             def retry_options
               {
                 retry_limit: 4,
-                retry_initial_wait_sec: 3,
+                retry_initial_wait_sec: 0,
               }
             end
           end

data/test/embulk/input/marketo_api/soap/test_lead.rb

@@ -47,12 +47,12 @@ module Embulk
 
               any_instance_of(Savon::Client) do |klass|
                 mock(klass).call(:get_multiple_leads, anything) do
-                  next_stream_leads_response
+                  savon_response(xml_lead_next)
                 end
               end
 
               proc = proc{ "" }
-              leads_count = next_stream_leads_response.xpath('//leadRecord').length
+              leads_count = savon_response(xml_lead_next).xpath('//leadRecord').length
               mock(proc).call(anything).times(leads_count)
 
               soap.each(timerange, {}, &proc)

data/test/lead_fixtures.rb

@@ -1,19 +1,11 @@
-module LeadFixtures
-  private
-
-  def leads_response
-    Nokogiri::XML(raw_response)
-  end
+require "savon_helper"
 
-  def next_stream_leads_response
-    Nokogiri::XML(raw_next_stream_response)
-  end
+module LeadFixtures
+  include SavonHelper
 
-  def preview_leads_response
-    Nokogiri::XML(raw_preview_response)
-  end
+  private
 
-  def leads(body)
+  def leads_xml(body)
     <<XML
 <?xml version="1.0" encoding="UTF-8"?>
 <SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
@@ -28,8 +20,8 @@ module LeadFixtures
 XML
   end
 
-  def raw_response
-    leads(<<XML)
+  def xml_lead_response
+    leads_xml(<<XML)
 <remainingCount>1</remainingCount>
 <newStreamPosition>#{stream_position}</newStreamPosition>
 <leadRecordList>
@@ -67,8 +59,8 @@ XML
     "next_steam_position"
   end
 
-  def raw_next_stream_response
-    leads(<<XML)
+  def xml_lead_next
+    leads_xml(<<XML)
 <returnCount>2</returnCount>
 <remainingCount>0</remainingCount>
 <newStreamPosition />
@@ -90,7 +82,7 @@ XML
 XML
   end
 
-  def raw_preview_response
+  def xml_lead_preview
     body = ""
     15.times do |i|
       body << <<XML
@@ -114,6 +106,6 @@ XML
 </leadRecordList>
 XML
     end
-    leads(body)
+    leads_xml(body)
   end
 end

data/test/savon_helper.rb

@@ -0,0 +1,17 @@
+module SavonHelper
+  private
+
+  def savon_response(body)
+    globals = {
+      namespace_identifier: :ns1,
+      env_namespace: 'SOAP-ENV',
+
+      # https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
+      strip_namespaces:         true,
+      convert_response_tags_to: lambda { |tag| tag.snakecase.to_sym},
+      convert_attributes_to:    lambda { |k,v| [k,v] },
+    }
+    httpi = HTTPI::Response.new(200, {}, body)
+    Savon::Response.new(httpi, globals, {advanced_typecasting: false})
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-30 00:00:00.000000000 Z
+date: 2016-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  name: perfect_retry
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -175,29 +189,6 @@ files:
 - README.md
 - Rakefile
 - embulk-input-marketo.gemspec
-- gemfiles/embulk-0.6.0-latest
-- gemfiles/embulk-0.6.13
-- gemfiles/embulk-0.6.14
-- gemfiles/embulk-0.6.15
-- gemfiles/embulk-0.6.16
-- gemfiles/embulk-0.6.17
-- gemfiles/embulk-0.6.18
-- gemfiles/embulk-0.6.19
-- gemfiles/embulk-0.6.20
-- gemfiles/embulk-0.6.21
-- gemfiles/embulk-0.6.22
-- gemfiles/embulk-0.6.23
-- gemfiles/embulk-0.6.24
-- gemfiles/embulk-0.6.25
-- gemfiles/embulk-0.6.26
-- gemfiles/embulk-0.6.27
-- gemfiles/embulk-0.7.0
-- gemfiles/embulk-0.7.0-latest
-- gemfiles/embulk-0.7.1
-- gemfiles/embulk-0.7.2
-- gemfiles/embulk-0.7.3
-- gemfiles/embulk-0.7.4
-- gemfiles/embulk-0.7.5
 - gemfiles/embulk-latest
 - gemfiles/template.erb
 - lib/embulk/input/marketo/activity_log.rb
@@ -220,6 +211,7 @@ files:
 - test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/savon_helper.rb
 homepage: https://github.com/treasure-data/embulk-input-marketo
 licenses:
 - Apache2
@@ -258,3 +250,4 @@ test_files:
 - test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
+- test/savon_helper.rb

data/gemfiles/embulk-0.6.0-latest

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "~> 0.6.0"

data/gemfiles/embulk-0.6.13

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.6.13"