embulk-input-marketo 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d395df72b2b9fafe6be7824001fb866c00f1f08
-  data.tar.gz: b344e828ecd3eb921ba38f9bc3c6bf26a3271ebe
+  metadata.gz: bcfd15d051ac950ef7529b31662348b01f6c5389
+  data.tar.gz: 1161b4da7811e5a0dd29454d5bd44b2ee6be8e73
 SHA512:
-  metadata.gz: 1893639b47f6268c154c4636fc432152b3a9061e2a8292edba2b03c0b32a51b85152f0d2c5f861ccb5404e4d5ca091670b92e411a9a25c3316470735483d2b07
-  data.tar.gz: c75fdc17d13700f6aef74022e9d5849777f5b1d861bd2a62ed2d24008c87245ee8cf7ed53c17acb432b8788222a9174a725ac11368571a88d482f355c9c55a5f
+  metadata.gz: 263729220d75599942c77113e4fed6f481453ed2b079fae30b26f63b12eb2e8ca96a667294ec378594241ae24d7e77a319ae9c5f9c6bb579c40fac58e90d6d4b
+  data.tar.gz: 5f8ee7bc9eaa36e1d08ced80eed97bb5bfd0e581250204a41369bfbe19eb240ad4c05ec6b2a2236bc4ea347f05daa3c1c690d4a0c51c20ff27c7b39f3fe5add6

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.2.0 - 2015-08-27
+
+This version breaks backword compatibility of marketo/lead. Please check README.md to modify your config.
+
+* [enhancement] Avoid timeout for marketo/lead input [#25](https://github.com/treasure-data/embulk-input-marketo/pull/25)
+* [fixed] Fix timestamp column [#24](https://github.com/treasure-data/embulk-input-marketo/pull/24)
+* [enhancement] Raise ConfigError if unretryable error occured [#23](https://github.com/treasure-data/embulk-input-marketo/pull/23)
+* [fixed] Fix the bug that the default value for wsdl is broken [#22](https://github.com/treasure-data/embulk-input-marketo/pull/22)
+
 ## 0.1.1 - 2015-08-19
 
 * [enhanement] Support scheduled execution [#20](https://github.com/treasure-data/embulk-input-marketo/pull/20) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/18). Thanks!]

data/README.md

@@ -18,7 +18,7 @@ This plugin uses Marketo SOAP API.
 Required Embulk version >= 0.6.13.
 
 * **Plugin type**: input
-* **Resume supported**: no
+* **Resume supported**: yes for `marketo/lead`, no for `marketo/activity_log`
 * **Cleanup supported**: no
 * **Guess supported**: yes
 
@@ -34,6 +34,19 @@ $ embulk gem install embulk-input-marketo
 
 Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 
+### market/lead
+
+**NOTE: If you use feature of scheduled execution (resume) with marketo/read, you should not specify until\_at because this plugin can't place new to_datetime (can't know the date to run with new config.**
+
+- **endpoint** SOAP endpoint URL for your account (string, required)
+- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
+- **user_id** Your user id (string, reqiured)
+- **encryption_key** Your encryption key (string, reqiured)
+- **from_datetime** Fetch leads since this time (string, required)
+- **to_datetime** Fetch leads until this time (string, default: Time.now)
+
+### market/activity_log
+
 - **endpoint** SOAP endpoint URL for your account (string, required)
 - **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
 - **user_id** Your user id (string, reqiured)
@@ -56,7 +69,7 @@ in:
   wsdl: https://wsdl-url.mktoapi.com/?WSDL
   user_id: user_ABC123
   encryption_key: TOPSECRET
-  last_updated_at: "2015-06-30"
+  from_datetime: "2015-06-30"
 out:
   type: stdout
 ```

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.1.1"
+  spec.version       = "0.2.0"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."

data/lib/embulk/input/marketo/base.rb

@@ -1,3 +1,4 @@
+require "embulk/input/marketo/timeslice"
 require "embulk/input/marketo_api"
 
 module Embulk
@@ -6,6 +7,8 @@ module Embulk
       class Base < InputPlugin
         PREVIEW_COUNT = 15
 
+        attr_reader :soap
+
         def self.target
           raise NotImplementedError
         end
@@ -46,7 +49,7 @@ module Embulk
         def self.soap_client(config)
           @soap ||=
             begin
-              endpoint_url = config.param(:endpoint, :string),
+              endpoint_url = config.param(:endpoint, :string)
               soap_config = {
                 endpoint_url: endpoint_url,
                 wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),

data/lib/embulk/input/marketo/lead.rb

@@ -4,6 +4,8 @@ module Embulk
   module Input
     module Marketo
       class Lead < Base
+        include Timeslice
+
         PREVIEW_COUNT = 15
 
         Plugin.register_input("marketo/lead", self)
@@ -12,13 +14,6 @@ module Embulk
           :lead
         end
 
-        def self.guess(config)
-          client = soap_client(config)
-          metadata = client.metadata
-
-          return {"columns" => generate_columns(metadata)}
-        end
-
         def self.generate_columns(metadata)
           columns = [
             {name: "id", type: "long"},
@@ -30,7 +25,7 @@ module Embulk
               case field[:data_type]
               when "integer"
                 "long"
-              when "dateTime", "date"
+              when "datetime", "date"
                 "timestamp"
               when "string", "text", "phone", "currency"
                 "string"
@@ -50,10 +45,23 @@ module Embulk
 
         def run
           count = 0
-          @soap.each(@last_updated_at) do |lead|
+          from_datetime = task[:from_datetime]
+          to_datetime = task[:to_datetime]
+          options = {}
+          options[:batch_size] = PREVIEW_COUNT if preview?
+
+          soap.each(from_datetime, to_datetime, options) do |lead|
             values = @columns.map do |column|
               name = column["name"].to_s
-              (lead[name] || {})[:value]
+              value = (lead[name] || {})[:value]
+              next unless value
+
+              case column["type"]
+              when "timestamp"
+                Time.parse(value)
+              else
+                value
+              end
             end
 
             page_builder.add(values)
@@ -64,7 +72,7 @@ module Embulk
 
           page_builder.finish
 
-          commit_report = {}
+          commit_report = {from_datetime: to_datetime}
           return commit_report
         end
       end

data/lib/embulk/input/marketo/timeslice.rb

@@ -0,0 +1,60 @@
+module Embulk
+  module Input
+    module Marketo
+      module Timeslice
+        def self.included(klass)
+          klass.extend ClassMethods
+        end
+
+        module ClassMethods
+          def guess(config)
+            if config.param(:last_updated_at, :string, default: nil)
+              Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
+            end
+
+            client = soap_client(config)
+            metadata = client.metadata
+
+            return {"columns" => generate_columns(metadata)}
+          end
+
+          def transaction(config, &control)
+            endpoint_url = config.param(:endpoint, :string)
+
+            if config.param(:last_updated_at, :string, default: nil)
+              Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
+            end
+
+            from_datetime = config.param(:from_datetime, :string)
+            to_datetime = config.param(:to_datetime, :string, default: Time.now.to_s)
+
+            if Time.parse(from_datetime) > Time.parse(to_datetime)
+              raise ConfigError, "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
+            end
+
+            task = {
+              endpoint_url: endpoint_url,
+              wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+              user_id: config.param(:user_id, :string),
+              encryption_key: config.param(:encryption_key, :string),
+              from_datetime: from_datetime,
+              to_datetime: to_datetime,
+              columns: config.param(:columns, :array)
+            }
+
+            columns = []
+
+            task[:columns].each do |column|
+              name = column["name"]
+              type = column["type"].to_sym
+
+              columns << Column.new(nil, name, type, column["format"])
+            end
+
+            resume(task, columns, 1, &control)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -53,7 +53,7 @@ module Embulk
           def fetch(request, options={}, &block)
             request[:batch_size] = options[:batch_size] || 100
 
-            response = savon.call(:get_lead_changes, message: request)
+            response = savon_call(:get_lead_changes, message: request)
             remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
             Embulk.logger.info "Remaining records: #{remaining}"
 

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -1,4 +1,5 @@
 require "savon"
+require "embulk/input/marketo_api/soap/timeslice"
 
 module Embulk
   module Input
@@ -38,6 +39,12 @@ module Embulk
             )
           end
 
+          def savon_call(*args)
+            catch_unretryable_error do
+              savon.call(*args)
+            end
+          end
+
           def signature
             timestamp = Time.now.to_s
             encryption_string = timestamp + user_id
@@ -48,6 +55,36 @@ module Embulk
               'requestSignature' => hashed_signature.to_s
             }
           end
+
+          def catch_unretryable_error(&block)
+            yield
+          rescue Savon::SOAPFault => e
+            Embulk.logger.debug "#{e.class}: #{e.to_hash}"
+            if e.to_hash[:fault][:faultcode].to_str == "SOAP-ENV:Client"
+              raise ConfigError, e.message
+            end
+          rescue Savon::HTTPError => e
+            # NOTE: Marketo API always return error as HTTP 500
+            # ref. https://jira.talendforge.org/secure/attachmentzip/unzip/167201/49761%5B1%5D/Marketo%20Enterprise%20API%202%200.pdf
+            Embulk.logger.debug "#{e.class}: #{e.http.body}"
+            soap_code = e.http.body[%r|<code>(.*?)</code>|, 1]
+            soap_message = e.http.body[%r|<message>(.*?)</message>|, 1]
+            case soap_code
+            when "10001", "20011"
+              # Internal Error
+              raise e
+            when "20015"
+              # Request Limit Exceeded
+              raise e
+            else
+              # unretryable error such as Authentication Failed, Invalid Request, etc.
+              raise ConfigError, soap_message
+            end
+          rescue SocketError => e
+            # maybe endpoint/wsdl domain was wrong
+            Embulk.logger.debug "SocketError: endpoint=#{endpoint} wsdl=#{wsdl}"
+            raise ConfigError, "SocketError: #{e.message} (endpoint is '#{endpoint}')"
+          end
         end
       end
     end

data/lib/embulk/input/marketo_api/soap/lead.rb

@@ -5,44 +5,55 @@ module Embulk
     module MarketoApi
       module Soap
         class Lead < Base
+          include Timeslice
+
+          # NOTE: batch_size is allowed at 1000, but that takes 2 minutes in 1 request.
+          #       We use 250 for the default (about 30 seconds)
+          BATCH_SIZE_DEFAULT = 250
+
           def metadata
             # http://developers.marketo.com/documentation/soap/describemobject/
-            response = savon.call(:describe_m_object, message: {object_name: "LeadRecord"})
+            response = savon_call(:describe_m_object, message: {object_name: "LeadRecord"})
             response.body[:success_describe_m_object][:result][:metadata][:field_list][:field]
           end
 
-          def each(last_updated_at, &block)
+          def each(from_datetime, to_datetime, options = {}, &block)
             # http://developers.marketo.com/documentation/soap/getmultipleleads/
+            to_datetime ||= Time.now
 
-            last_updated_at = Time.parse(last_updated_at).iso8601
-
-            # TODO: generate request in #fetch
-            # TODO: use PREVIEW_COUNT as batch_size in preview
-            request = {
-              lead_selector: {
-                oldest_updated_at: last_updated_at,
-              },
-              attributes!: {
-                lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
-              },
-              batch_size: 1000,
-            }
+            generate_time_range(from_datetime, to_datetime).each do |range|
+              request = {
+                lead_selector: {
+                  oldest_updated_at: range[:from].iso8601,
+                  latest_updated_at: range[:to].iso8601,
+                },
+                attributes!: {
+                  lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
+                },
+                batch_size: options[:batch_size] || BATCH_SIZE_DEFAULT,
+              }
+              Embulk.logger.info "Fetching from '#{range[:from]}' to '#{range[:to]}'..."
 
-            stream_position = fetch(request, &block)
+              stream_position = fetch(request, &block)
 
-            while stream_position
-              stream_position = fetch(request.merge(stream_position: stream_position), &block)
+              while stream_position
+                stream_position = fetch(request.merge(stream_position: stream_position), &block)
+              end
             end
           end
 
           private
 
           def fetch(request = {}, &block)
-            response = savon.call(:get_multiple_leads, message: request)
+            start = Time.now
+            response = savon_call(:get_multiple_leads, message: request)
+            Embulk.logger.info "Fetched in #{Time.now - start} seconds"
 
+            records = response.xpath('//leadRecordList/leadRecord')
             remaining = response.xpath('//remainingCount').text.to_i
-            Embulk.logger.info "Remaining records: #{remaining}"
-            response.xpath('//leadRecordList/leadRecord').each do |lead|
+            Embulk.logger.info "Fetched records in the range: #{records.size}"
+            Embulk.logger.info "Remaining records in the range: #{remaining}"
+            records.each do |lead|
               record = {
                 "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
                 "email" => {type: :string, value: lead.xpath('Email').text}

data/lib/embulk/input/marketo_api/soap/timeslice.rb

@@ -0,0 +1,44 @@
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        module Timeslice
+          private
+
+          def generate_time_range(from, to)
+            # e.g. from = 2010-01-01 15:00, to = 2010-01-03 09:30
+            # convert to such array:
+            # [
+            #   {from: 2010-01-01 15:00, to: 2010-01-01 16:00},
+            #   {from: 2010-01-01 16:00, to: 2010-01-01 17:00},
+            #   ...
+            #   {from: 2010-01-03 08:00, to: 2010-01-03 09:00},
+            #   {from: 2010-01-03 09:00, to: 2010-01-03 09:30},
+            # ]
+            # to fetch data from Marketo API with each day as
+            # desribed on official blog:
+            # http://developers.marketo.com/blog/performance-tuning-api-requests/
+            to ||= Time.now
+            from = Time.parse(from) unless from.is_a?(Time)
+            to = Time.parse(to) unless to.is_a?(Time)
+
+            result = []
+            since = from
+            while since < to
+              next_since = since + 3600
+              if to < next_since
+                next_since = to
+              end
+              result << {
+                from: since,
+                to: next_since
+              }
+              since = next_since
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/marketo/test_base.rb

@@ -32,6 +32,29 @@ module Embulk
           assert_equal(next_config_diff, actual)
         end
 
+        class SoapClientTest < self
+          def setup
+            stub(Base).target { :lead }
+            @client = Base.soap_client(config)
+          end
+
+          def test_endpoint
+            assert_equal(settings[:endpoint], @client.endpoint)
+          end
+
+          def test_wsdl
+            assert_equal(settings[:wsdl], @client.wsdl)
+          end
+
+          def test_user_id
+            assert_equal(settings[:user_id], @client.user_id)
+          end
+
+          def test_encryption_key
+            assert_equal(settings[:encryption_key], @client.encryption_key)
+          end
+        end
+
         private
 
         def config

data/test/embulk/input/marketo/test_lead.rb

@@ -1,5 +1,6 @@
 require "prepare_embulk"
 require "lead_fixtures"
+require "mute_logger"
 require "embulk/input/marketo/lead"
 
 module Embulk
@@ -7,6 +8,7 @@ module Embulk
     module Marketo
       class LeadTest < Test::Unit::TestCase
         include LeadFixtures
+        include MuteLogger
 
         def test_target
           assert_equal(:lead, Lead.target)
@@ -21,7 +23,28 @@ module Embulk
         def setup_plugin
           @page_builder = Object.new
           @plugin = Lead.new(task, nil, nil, @page_builder)
-          stub(Embulk).logger { ::Logger.new(File::NULL) }
+          mute_logger
+        end
+
+        def test_invalid_from_datetime_to_datetime
+          control = proc {} # dummy
+
+          settings = {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ],
+            from_datetime: Time.now + 3600,
+            to_datetime: Time.now,
+          }
+          config = DataSource[settings.to_a]
+
+          assert_raise(ConfigError) do
+            Lead.transaction(config, &control)
+          end
         end
 
         class RunTest < self
@@ -51,11 +74,20 @@ module Embulk
             @plugin.run
           end
 
+          def test_run_commit_report
+            # do not requests
+            stub(@page_builder).finish
+            stub(@plugin.soap).each { }
+
+            commit_report = @plugin.run
+            assert_equal to_datetime, commit_report[:from_datetime]
+          end
+
           def test_preview_through
             stub(@plugin).preview? { true }
 
             any_instance_of(Savon::Client) do |klass|
-              mock(klass).call(:get_multiple_leads, message: request) do
+              mock(klass).call(:get_multiple_leads, message: request.merge(batch_size: Lead::PREVIEW_COUNT)) do
                 preview_leads_response
               end
             end
@@ -68,13 +100,75 @@ module Embulk
             @plugin.run
           end
 
+          class SavonCallTest < self
+            def test_soap_error
+              assert_raise(Embulk::ConfigError) do
+                @soap.send(:catch_unretryable_error) do
+                  raise Savon::SOAPFault.new(nil, Nori.new(default_nori_options), xml)
+                end
+              end
+            end
+
+            def test_http_error_on_client
+              assert_raise(Embulk::ConfigError) do
+                @soap.send(:catch_unretryable_error) do
+                  raise Savon::HTTPError.new(HTTPI::Response.new(500, {}, xml("20000")))
+                end
+              end
+            end
+
+            def test_http_error_on_server
+              assert_raise(Savon::HTTPError) do
+                @soap.send(:catch_unretryable_error) do
+                  # Internal Error
+                  raise Savon::HTTPError.new(HTTPI::Response.new(500, {}, xml("10001")))
+                end
+              end
+            end
+
+            def test_socket_error
+              stub(@soap).endpoint { "http://192.0.2.0/" }
+
+              assert_raise(Embulk::ConfigError) do
+                @plugin.run
+              end
+            end
+
+            def xml(code = nil, message = nil)
+              <<-XML
+<?xml version="1.0" encoding="UTF-8"?>
+<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/"><SOAP-ENV:Body><SOAP-ENV:Fault>
+<faultcode>SOAP-ENV:Client</faultcode>
+<faultstring>#{message || "20014 - Authentication failed"}</faultstring>
+<detail>
+<ns1:serviceException xmlns:ns1="http://www.marketo.com/mktows/">
+<name>mktServiceException</name>
+<message>#{message || "Authentication failed (20014)"}</message>
+<code>#{code || "20014"}</code>
+</ns1:serviceException></detail></SOAP-ENV:Fault></SOAP-ENV:Body></SOAP-ENV:Envelope>
+              XML
+            end
+
+            def default_nori_options
+              # https://github.com/savonrb/savon/blob/v2.11.1/lib/savon/options.rb#L75-L94
+              {
+                :strip_namespaces          => true,
+                :convert_tags_to  => lambda { |tag| tag.snakecase.to_sym},
+                :convert_attributes_to     => lambda { |k,v| [k,v] },
+              }
+            end
+          end
+
           private
 
           def request
             {
-              lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
+              lead_selector: {
+                oldest_updated_at: timerange.first[:from].iso8601,
+                latest_updated_at: timerange.first[:to].iso8601,
+              },
               attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
-              batch_size: 1000
+              batch_size: MarketoApi::Soap::Lead::BATCH_SIZE_DEFAULT,
             }
           end
         end
@@ -114,24 +208,35 @@ module Embulk
             wsdl: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
+            to_datetime: to_datetime,
             columns: [
               {"name" => "Name", "type" => "string"},
             ]
           }
         end
 
-        def last_updated_at
+        def from_datetime
           "2015-07-01 00:00:00+00:00"
         end
 
+        def to_datetime
+          "2015-07-01 00:00:05+00:00"
+        end
+
+        def timerange
+          soap = MarketoApi::Soap::Lead.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+          soap.send(:generate_time_range, from_datetime, to_datetime)
+        end
+
         def task
           {
             endpoint_url: "https://marketo.example.com",
             wsdl_url: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
+            to_datetime: to_datetime,
             columns: [
               {"name" => "Name", "type" => "string"},
             ]
@@ -163,7 +268,7 @@ module Embulk
           [
             {name: "id", type: "long"},
             {name: "email", type: "string"},
-            {name: "FieldName", type: "string"},
+            {name: "FieldName", type: "timestamp"},
           ]
         end
       end

data/test/embulk/input/marketo_api/soap/test_lead.rb

@@ -1,5 +1,6 @@
 require "embulk/input/marketo_api/soap/lead"
 require "lead_fixtures"
+require "mute_logger"
 
 module Embulk
   module Input
@@ -7,32 +8,97 @@ module Embulk
       module Soap
         class LeadTest < Test::Unit::TestCase
           include LeadFixtures
+          include MuteLogger
 
-          def test_each
-            stub(Embulk).logger { ::Logger.new(IO::NULL) }
-            last_updated_at = "2015-07-06"
+          def setup
+            mute_logger
+          end
 
-            request = {
-              lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
-              attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
-              batch_size: 1000
-            }
+          class TestEach < self
+            def setup
+              super
+            end
+
+            def test_each_invoke_fetch
+              from_datetime = "2015-07-06"
+              to_datetime = "2015-07-07"
+              timerange = soap.send(:generate_time_range, from_datetime, to_datetime)
+
+              stub(soap).fetch { nil }
+              mock(soap).fetch(anything).times(timerange.length)
+
+              soap.each(from_datetime, to_datetime) { }
+            end
+
+            def test_each_invoke_fetch_with_specified_time
+              from_datetime = "2015-07-06"
+              to_datetime = "2015-07-07"
+              timerange = soap.send(:generate_time_range, from_datetime, to_datetime)
+
+              request = {
+                lead_selector: {
+                  oldest_updated_at: timerange.first[:from].iso8601,
+                  latest_updated_at: timerange.first[:to].iso8601,
+                },
+                attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
+                batch_size: Lead::BATCH_SIZE_DEFAULT,
+              }
+
+              stub(soap).fetch { nil }
+              mock(soap).fetch(request)
+
+              soap.each(from_datetime, to_datetime) { }
+            end
 
-            any_instance_of(Savon::Client) do |klass|
-              mock(klass).call(:get_multiple_leads, message: request) do
-                next_stream_leads_response
+            def test_each_fetch_next_page
+              from_datetime = "2015-07-06 00:00:00"
+              to_datetime = "2015-07-06 00:00:01"
+
+              any_instance_of(Savon::Client) do |klass|
+                mock(klass).call(:get_multiple_leads, anything) do
+                  next_stream_leads_response
+                end
               end
+
+              proc = proc{ "" }
+              leads_count = next_stream_leads_response.xpath('//leadRecord').length
+              mock(proc).call(anything).times(leads_count)
+
+              soap.each(from_datetime, to_datetime, &proc)
             end
+          end
 
-            proc = proc{ "" }
-            leads_count = next_stream_leads_response.xpath('//leadRecord').length
-            mock(proc).call(anything).times(leads_count)
+          class TestGenerateTime < self
+            def setup
+              mute_logger
+            end
 
-            soap.each(last_updated_at, &proc)
+            data do
+              {
+                "8/1 to 8/2" => ["2015-08-01 00:00:00", "2015-08-02 00:00:00", 24],
+                "over the days" => ["2015-08-01 19:00:00", "2015-08-03 05:00:00", 34],
+                "odd times" => ["2015-08-01 11:11:11", "2015-08-01 22:22:22", 12],
+              }
+            end
+            def test_generate_time_range_by_1hour(data)
+              from, to, count = data
+              range = soap.send(:generate_time_range, from, to)
+              assert_equal count, range.length
+            end
+
+            def test_if_to_is_nil_use_time_now
+              from = "2000-01-01"
+              now = Time.now
+              stub(Time).now { now }
+
+              range = soap.send(:generate_time_range, from, nil)
+              assert_equal now, range.last[:to]
+            end
           end
 
           class TestMetadata < self
             def setup
+              super
               @savon = soap.__send__(:savon)
               stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
             end

data/test/mute_logger.rb

@@ -0,0 +1,7 @@
+module MuteLogger
+  private
+
+  def mute_logger
+    stub(Embulk).logger { ::Logger.new(IO::NULL) }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -175,10 +175,12 @@ files:
 - lib/embulk/input/marketo/activity_log.rb
 - lib/embulk/input/marketo/base.rb
 - lib/embulk/input/marketo/lead.rb
+- lib/embulk/input/marketo/timeslice.rb
 - lib/embulk/input/marketo_api.rb
 - lib/embulk/input/marketo_api/soap/activity_log.rb
 - lib/embulk/input/marketo_api/soap/base.rb
 - lib/embulk/input/marketo_api/soap/lead.rb
+- lib/embulk/input/marketo_api/soap/timeslice.rb
 - test/activity_log_fixtures.rb
 - test/embulk/input/marketo/test_activity_log.rb
 - test/embulk/input/marketo/test_base.rb
@@ -188,6 +190,7 @@ files:
 - test/embulk/input/marketo_api/soap/test_lead.rb
 - test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
+- test/mute_logger.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
 homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -224,5 +227,6 @@ test_files:
 - test/embulk/input/marketo_api/soap/test_lead.rb
 - test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
+- test/mute_logger.rb
 - test/prepare_embulk.rb
 - test/run-test.rb