embulk-input-marketo 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 803b16c2edca4432ffcb1f32e9914d944b3747dd
-  data.tar.gz: bb28951330b3b3ac1a8d3bf7a3e81541915268e2
+  metadata.gz: ab9b973d17f88ee84d60abec578abaf3734ef9db
+  data.tar.gz: 2875742fde0e64de55ea08458674f041b1ed2030
 SHA512:
-  metadata.gz: 53f1320084a90a824af96b5205ae05f3d8596e79169abc7b15282c67d359ca8c26b8c1e433dcab5ff7b678ba1b239347b7ebd9eda547eb79bbde6ca931227aed
-  data.tar.gz: e6a1e61e3a7d4d0b875b0e704298d7e96d0e1ab395125e461ef704ef069267ac407b515538e2576dbf7ada5420b2f8b473a5d04337f5600648dd5e3649d2c7f6
+  metadata.gz: 2b559f997b7f9e8cab9b1caec4e65fb21edce75665e1162a8ae93a2709b11cb9317c955320c0bd206d9b049a4f0d0f61157b2b86452bafe1f89bb2206de6eab5
+  data.tar.gz: 5df8a59649e0bdb9b1b607b983d1fa5cc35fa0108481d20e1aa195e052dc70b34779e572f793175de070bd90f96fd95dbf497965acef93ebfc8e55a09380bcee

data/.ruby-version

@@ -1 +1 @@
-jruby-1.7.20
+jruby-9.0.1.0

data/.travis.yml

@@ -1,16 +1,34 @@
 language: ruby
-rvm:
-  - jruby-19mode
+
 addons:
   code_climate:
     repo_token:
       secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
+
 notifications:
   slack:
     secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
-jdk:
-  - oraclejdk8
+
+before_install:
+  - |
+    # Currently, Travis can't treat jruby 9.0.1.0
+    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
+      rvm get head
+      rvm use jruby-9.0.1.0 --install
+      ruby -v
+    fi
+
+jdk: oraclejdk8
+
+rvm:
+  - jruby-19mode
+  - jruby-9.0.0.0
+
 gemfile:
+  - gemfiles/embulk-0.6.0-latest
+  - gemfiles/embulk-0.6.13
+  - gemfiles/embulk-0.6.14
+  - gemfiles/embulk-0.6.15
   - gemfiles/embulk-0.6.16
   - gemfiles/embulk-0.6.17
   - gemfiles/embulk-0.6.18
@@ -22,8 +40,52 @@ gemfile:
   - gemfiles/embulk-0.6.24
   - gemfiles/embulk-0.6.25
   - gemfiles/embulk-0.6.26
+  - gemfiles/embulk-0.6.27
+  - gemfiles/embulk-0.7.0
+  - gemfiles/embulk-0.7.0-latest
+  - gemfiles/embulk-0.7.1
+  - gemfiles/embulk-0.7.2
+  - gemfiles/embulk-0.7.3
+  - gemfiles/embulk-0.7.4
+  - gemfiles/embulk-0.7.5
   - gemfiles/embulk-latest
+
 matrix:
+  exclude:
+    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
+  include:
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.0-latest}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.13}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.14}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.15}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.16}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.17}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.18}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.19}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.20}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.21}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.22}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.23}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.24}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.25}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.26}
+    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.27}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0-latest}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.1}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.2}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.3}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.4}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.5}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-latest}
+    
+
   allow_failures:
     - gemfile: gemfiles/embulk-0.6.22
-    - gemfile: gemfiles/embulk-latest
+    - gemfile: gemfiles/embulk-0.7.0
+    - gemfile: gemfiles/embulk-0.7.1
+    # Ignore failure for *-latest
+    - gemfile: embulk-0.6.0-latest
+    - gemfile: embulk-0.7.0-latest
+    - gemfile: embulk-latest
+    

data/.travis.yml.erb

@@ -0,0 +1,47 @@
+language: ruby
+
+addons:
+  code_climate:
+    repo_token:
+      secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
+
+notifications:
+  slack:
+    secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
+
+before_install:
+  - |
+    # Currently, Travis can't treat jruby 9.0.1.0
+    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
+      rvm get head
+      rvm use jruby-9.0.1.0 --install
+      ruby -v
+    fi
+
+jdk: oraclejdk8
+
+rvm:
+  - jruby-19mode
+  - jruby-9.0.0.0
+
+gemfile:
+<% versions.each do |file| -%>
+  - gemfiles/<%= file %>
+<% end -%>
+
+matrix:
+  exclude:
+    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
+  include:
+    <% matrix.each do |m| -%>
+<%= m %>
+    <% end %>
+
+  allow_failures:
+    - gemfile: gemfiles/embulk-0.6.22
+    - gemfile: gemfiles/embulk-0.7.0
+    - gemfile: gemfiles/embulk-0.7.1
+    # Ignore failure for *-latest
+    <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
+- gemfile: <%= file %>
+    <% end %>

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.1 - 2015-10-06
+
+* [enhancement] Supports embulk.0.7 [#43](https://github.com/treasure-data/embulk-input-marketo/pull/43)
+* [maintenance] Refactor [#40](https://github.com/treasure-data/embulk-input-marketo/pull/40)
+
 ## 0.3.0 - 2015-09-30
 
 This version breaks backword compatibility of marketo/activity_log. Please check README.md to modify your config.

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.3.0"
+  spec.version       = "0.3.1"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."

data/gemfiles/embulk-0.6.0-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "~> 0.6.0"

data/gemfiles/embulk-0.6.13

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.13"

data/gemfiles/embulk-0.6.14

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.14"

data/gemfiles/embulk-0.6.15

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.15"

data/gemfiles/embulk-0.6.27

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.27"

data/gemfiles/embulk-0.7.0

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.0"

data/gemfiles/embulk-0.7.0-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "~> 0.7.0"

data/gemfiles/embulk-0.7.1

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.1"

data/gemfiles/embulk-0.7.2

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.2"

data/gemfiles/embulk-0.7.3

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.3"

data/gemfiles/embulk-0.7.4

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.4"

data/gemfiles/embulk-0.7.5

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.7.5"

data/gemfiles/embulk-latest

@@ -1,4 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
 
-gem "embulk", "> 0.6.16"
+gem "embulk", "> 0.6.13"

data/lib/embulk/input/marketo/activity_log.rb

@@ -4,12 +4,23 @@ module Embulk
   module Input
     module Marketo
       class ActivityLog < Base
+        BATCH_SIZE_DEFAULT = 100
+
         Plugin.register_input("marketo/activity_log", self)
 
         def self.target
           :activity_log
         end
 
+        def self.resume(task, columns, count, &control)
+          task_reports = yield(task, columns, count)
+
+          # NOTE: If this plugin supports to run by multi threads, this
+          # implementation is terrible.
+          next_config_diff = task_reports.first
+          return next_config_diff
+        end
+
         def self.transaction(config, &control)
           endpoint_url = config.param(:endpoint, :string)
 
@@ -25,16 +36,7 @@ module Embulk
             columns: config.param(:columns, :array)
           }
 
-          columns = []
-
-          task[:columns].each do |column|
-            name = column["name"]
-            type = column["type"].to_sym
-
-            columns << Column.new(nil, name, type, column["format"])
-          end
-
-          resume(task, columns, 1, &control)
+          resume(task, embulk_columns(config), 1, &control)
         end
 
         def self.guess(config)
@@ -51,46 +53,37 @@ module Embulk
           return {"columns" => columns}
         end
 
-        def run
-          if preview?
-            batch_size = PREVIEW_COUNT
-          else
-            batch_size = 100
-          end
+        def init
+          @columns = task[:columns]
+          @soap = MarketoApi.soap_client(task, target)
+        end
 
-          count = 0
+        def run
+          options = {
+            to: task[:to_datetime],
+            batch_size: (preview? ? PREVIEW_COUNT : BATCH_SIZE_DEFAULT),
+          }
 
-          last_updated_at = @soap.each(task[:from_datetime], batch_size: batch_size, to: task[:to_datetime]) do |activity_log|
+          counter = 0
+          latest_updated_at = @soap.each(task[:from_datetime], options) do |activity_log|
             values = @columns.map do |column|
               name = column["name"].to_s
               value = activity_log[name]
-              next unless value
-
-              case column["type"].to_s
-              when "timestamp"
-                begin
-                  Time.parse(value)
-                rescue => e
-                  raise ConfigError, "Can't parse as Time '#{value}' (column is #{column["name"]})"
-                end
-              else
-                value
-              end
+              cast_value(column, value)
             end
 
             page_builder.add(values)
-            count += 1
-            break if preview? && count >= PREVIEW_COUNT
+            break if preview? && (counter += 1) >= PREVIEW_COUNT
           end
 
           page_builder.finish
 
-          commit_report = {}
-          if !preview? && last_updated_at
-            commit_report = {from_datetime: last_updated_at}
+          task_report = {}
+          if !preview? && latest_updated_at
+            task_report = {from_datetime: latest_updated_at}
           end
 
-          return commit_report
+          return task_report
         end
       end
     end

data/lib/embulk/input/marketo/base.rb

@@ -1,4 +1,3 @@
-require "embulk/input/marketo/timeslice"
 require "embulk/input/marketo_api"
 
 module Embulk
@@ -13,15 +12,6 @@ module Embulk
           raise NotImplementedError
         end
 
-        def self.resume(task, columns, count, &control)
-          commit_reports = yield(task, columns, count)
-
-          # NOTE: If this plugin supports to run by multi threads, this
-          # implementation is terrible.
-          next_config_diff = commit_reports.first
-          return next_config_diff
-        end
-
         def self.soap_client(config)
           @soap ||=
             begin
@@ -37,26 +27,15 @@ module Embulk
             end
         end
 
-        def init
-          @last_updated_at = task[:last_updated_at]
-          @columns = task[:columns]
-          @soap = MarketoApi.soap_client(task, target)
-        end
-
-        private
+        def self.embulk_columns(config)
+          config.param(:columns, :array).map do |column|
+            name = column["name"]
+            type = column["type"].to_sym
 
-        def preview?
-          begin
-            org.embulk.spi.Exec.isPreview()
-          rescue java.lang.NullPointerException => e
-            false
+            Column.new(nil, name, type, column["format"])
           end
         end
 
-        def target
-          self.class.target
-        end
-
         def self.format_range(config)
           if config.param(:last_updated_at, :string, default: nil)
             Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
@@ -71,11 +50,11 @@ module Embulk
             Time.parse(to_datetime)
           rescue => e
             # possibly Time.parse fail
-            raise ConfigError, e.message
+            raise ConfigError.new e.message
           end
 
           if Time.parse(from_datetime) > Time.parse(to_datetime)
-            raise ConfigError, "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
+            raise ConfigError.new "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
           end
 
           {
@@ -83,6 +62,72 @@ module Embulk
             to: to_datetime,
           }
         end
+
+        def self.timeslice(from, to, count)
+          generate_time_range(from, to).each_slice(count).to_a
+        end
+
+        def self.generate_time_range(from, to)
+          # e.g. from = 2010-01-01 15:00, to = 2010-01-03 09:30
+          # convert to such array:
+          # [
+          #   {from: 2010-01-01 15:00, to: 2010-01-01 16:00},
+          #   {from: 2010-01-01 16:00, to: 2010-01-01 17:00},
+          #   ...
+          #   {from: 2010-01-03 08:00, to: 2010-01-03 09:00},
+          #   {from: 2010-01-03 09:00, to: 2010-01-03 09:30},
+          # ]
+          # to fetch data from Marketo API with each day as
+          # desribed on official blog:
+          # http://developers.marketo.com/blog/performance-tuning-api-requests/
+          to ||= Time.now
+          from = Time.parse(from) unless from.is_a?(Time)
+          to = Time.parse(to) unless to.is_a?(Time)
+
+          result = []
+          since = from
+          while since < to
+            next_since = since + 3600
+            if to < next_since
+              next_since = to
+            end
+            result << {
+              "from" => since,
+              "to" => next_since
+            }
+            since = next_since
+          end
+          result
+        end
+
+        private
+
+        def preview?
+          begin
+            org.embulk.spi.Exec.isPreview()
+          rescue java.lang.NullPointerException => e
+            false
+          end
+        end
+
+        def cast_value(column, value)
+          return unless value
+
+          case column["type"].to_s
+          when "timestamp"
+            begin
+              Time.parse(value)
+            rescue => e
+              raise ConfigError.new "Can't parse as Time '#{value}' (column is #{column["name"]})"
+            end
+          else
+            value
+          end
+        end
+
+        def target
+          self.class.target
+        end
       end
     end
   end

data/lib/embulk/input/marketo/lead.rb

@@ -4,7 +4,7 @@ module Embulk
   module Input
     module Marketo
       class Lead < Base
-        include Timeslice
+        TIMESLICE_COUNT_PER_TASK = 24
 
         Plugin.register_input("marketo/lead", self)
 
@@ -12,6 +12,46 @@ module Embulk
           :lead
         end
 
+        def self.guess(config)
+          if config.param(:last_updated_at, :string, default: nil)
+            Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
+          end
+
+          client = soap_client(config)
+          metadata = client.metadata
+
+          return {"columns" => generate_columns(metadata)}
+        end
+
+        def self.resume(task, columns, count, &control)
+          task_reports = yield(task, columns, count)
+
+          # When no task ran, task_reports is empty
+          return {} if task_reports.empty?
+          # all task returns same report as {from_datetime: to_datetime}
+          return task_reports.first
+        end
+
+        def self.transaction(config, &control)
+          endpoint_url = config.param(:endpoint, :string)
+
+          range = format_range(config)
+
+          ranges = timeslice(range[:from], range[:to], TIMESLICE_COUNT_PER_TASK)
+          task = {
+            endpoint_url: endpoint_url,
+            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+            user_id: config.param(:user_id, :string),
+            encryption_key: config.param(:encryption_key, :string),
+            from_datetime: range[:from],
+            to_datetime: range[:to],
+            ranges: ranges,
+            columns: config.param(:columns, :array)
+          }
+
+          resume(task, embulk_columns(config), ranges.size, &control)
+        end
+
         def self.generate_columns(metadata)
           columns = [
             {name: "id", type: "long"},
@@ -48,41 +88,29 @@ module Embulk
         end
 
         def run
-          from_datetime = task[:from_datetime]
-          to_datetime = task[:to_datetime] || Time.now
-
           options = {}
           options[:batch_size] = PREVIEW_COUNT if preview?
 
+          counter = 0
           @ranges.each do |range|
             soap.each(range, options) do |lead|
               values = @columns.map do |column|
                 name = column["name"].to_s
                 value = (lead[name] || {})[:value]
-                next unless value
-
-                case column["type"]
-                when "timestamp"
-                  begin
-                    Time.parse(value)
-                  rescue => e
-                    raise ConfigError, "Can't parse as Time '#{value}' (column is #{column["name"]})"
-                  end
-                else
-                  value
-                end
+                cast_value(column, value)
               end
 
               page_builder.add(values)
+              break if preview? && (counter += 1) >= PREVIEW_COUNT
             end
           end
 
           page_builder.finish
 
-          commit_report = {
-            from_datetime: to_datetime
+          task_report = {
+            from_datetime: task[:to_datetime]
           }
-          return commit_report
+          return task_report
         end
       end
     end

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -5,30 +5,29 @@ module Embulk
     module MarketoApi
       module Soap
         class ActivityLog < Base
-          def metadata(last_updated_at, options={})
+          def metadata(from_datetime, options={})
             activity_logs = []
 
-            fetch_by_last_updated_at(last_updated_at, options) do |record|
+            fetch_by_from_datetime(from_datetime, options) do |record|
               activity_logs << record
             end
 
             Guess::SchemaGuess.from_hash_records(activity_logs)
           end
 
-          def each(last_updated_at, options={}, &block)
-            response = fetch_by_last_updated_at(last_updated_at, options, &block)
+          def each(from_datetime, options={}, &block)
+            response = fetch_by_from_datetime(from_datetime, options, &block)
             while response[:remaining_count] > 0 do
-              response = fetch_by_last_updated_at(last_updated_at, options.merge(offset: response[:offset]), &block)
+              response = fetch_by_from_datetime(from_datetime, options.merge(offset: response[:offset]), &block)
             end
 
-            response[:last_updated_at]
+            response[:from_datetime]
           end
 
           private
 
-          def fetch_by_last_updated_at(last_updated_at, options={}, &block)
-            last_updated_at = last_updated_at.to_s
-            last_updated_at = Time.parse(last_updated_at).iso8601
+          def fetch_by_from_datetime(from_datetime, options={}, &block)
+            from = Time.parse(from_datetime.to_s).iso8601
 
             to =
               if options[:to]
@@ -39,7 +38,7 @@ module Embulk
 
             request = {
               start_position: {
-                oldest_created_at: last_updated_at,
+                oldest_created_at: from,
                 latest_created_at: to,
               },
               batch_size: options[:batch_size] || 100
@@ -58,7 +57,7 @@ module Embulk
 
             if activities_list.nil?
               Embulk.logger.info "No record is fetched."
-              return {remaining_count: 0, offset: nil, last_updated_at: nil}
+              return {remaining_count: 0, offset: nil, from_datetime: nil}
             end
 
             activities = activities_list[:lead_change_record].sort_by { |activity| Time.parse(activity[:activity_date_time]) }
@@ -85,7 +84,7 @@ module Embulk
             {
               remaining_count: remaining,
               offset: response.body[:success_get_lead_changes][:result][:new_start_position][:offset],
-              last_updated_at: activities.last[:activity_date_time]
+              from_datetime: activities.last[:activity_date_time]
             }
           end
         end

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -77,7 +77,7 @@ module Embulk
           rescue Savon::SOAPFault => e
             Embulk.logger.debug "#{e.class}: #{e.to_hash}"
             if e.to_hash[:fault][:faultcode].to_str == "SOAP-ENV:Client"
-              raise ConfigError, e.message
+              raise ConfigError.new e.message
             end
           rescue Savon::HTTPError => e
             # NOTE: Marketo API always return error as HTTP 500
@@ -94,12 +94,12 @@ module Embulk
               raise e
             else
               # unretryable error such as Authentication Failed, Invalid Request, etc.
-              raise ConfigError, soap_message
+              raise ConfigError.new soap_message
             end
           rescue SocketError, Errno::ECONNREFUSED => e
             # maybe endpoint/wsdl domain was wrong
             Embulk.logger.debug "Connection error: endpoint=#{endpoint} wsdl=#{wsdl}"
-            raise ConfigError, "Connection error: #{e.message} (endpoint is '#{endpoint}')"
+            raise ConfigError.new "Connection error: #{e.message} (endpoint is '#{endpoint}')"
           end
         end
       end

data/test/embulk/input/marketo/test_activity_log.rb

@@ -1,4 +1,5 @@
 require "prepare_embulk"
+require "override_assert_raise"
 require "embulk/input/marketo/activity_log"
 require "activity_log_fixtures"
 
@@ -7,6 +8,7 @@ module Embulk
     module Marketo
       class ActivityLogTest < Test::Unit::TestCase
         include ActivityLogFixtures
+        include OverrideAssertRaise
 
         class TransactionTest < self
           def test_generate_task
@@ -19,6 +21,17 @@ module Embulk
             ActivityLog.transaction(config, &control)
           end
 
+          def test_resume
+            next_config_diff = {from_datetime: from_datetime}
+            control = proc { [next_config_diff] } # In actual, embulk prepares control block returning Array.
+            columns = task[:columns].map do |col|
+              Column.new(nil, col["name"], col["type"].to_sym)
+            end
+
+            actual = ActivityLog.resume(task, columns, 1, &control)
+            assert_equal(next_config_diff, actual)
+          end
+
           private
 
           def settings

data/test/embulk/input/marketo/test_base.rb

@@ -11,17 +11,6 @@ module Embulk
           end
         end
 
-        def test_resume
-          next_config_diff = {last_updated_at: last_updated_at}
-          control = proc { [next_config_diff] } # In actual, embulk prepares control block returning Array.
-          columns = task[:columns].map do |col|
-            Column.new(nil, col["name"], col["type"].to_sym)
-          end
-
-          actual = Base.resume(task, columns, 1, &control)
-          assert_equal(next_config_diff, actual)
-        end
-
         class SoapClientTest < self
           def setup
             stub(Base).target { :lead }
@@ -57,14 +46,14 @@ module Embulk
             wsdl: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
             columns: [
               {"name" => "Name", "type" => "string"},
             ]
           }
         end
 
-        def last_updated_at
+        def from_datetime
           "2015-07-01 00:00:00+00:00"
         end
 
@@ -74,7 +63,7 @@ module Embulk
             wsdl_url: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
             columns: [
               {"name" => "Name", "type" => "string"},
             ]

data/test/embulk/input/marketo/test_lead.rb

@@ -1,6 +1,7 @@
 require "prepare_embulk"
 require "lead_fixtures"
 require "mute_logger"
+require "override_assert_raise"
 require "embulk/input/marketo/lead"
 
 module Embulk
@@ -9,6 +10,7 @@ module Embulk
       class LeadTest < Test::Unit::TestCase
         include LeadFixtures
         include MuteLogger
+        include OverrideAssertRaise
 
         def test_target
           assert_equal(:lead, Lead.target)

data/test/embulk/input/marketo_api/soap/test_activity_log.rb

@@ -15,7 +15,7 @@ module Embulk
           def test_each
             request = {
               start_position: {
-                oldest_created_at: Time.parse(last_updated_at).iso8601,
+                oldest_created_at: Time.parse(from_datetime).iso8601,
               },
               batch_size: 100
             }
@@ -31,13 +31,13 @@ module Embulk
             last_activity_log = activity_logs.last
 
             mock(proc).call(anything).times(activity_logs.size)
-            assert_equal(last_activity_log[:activity_date_time], soap.each(last_updated_at, &proc))
+            assert_equal(last_activity_log[:activity_date_time], soap.each(from_datetime, &proc))
           end
 
           def test_each_with_no_response
             request = {
               start_position: {
-                oldest_created_at: Time.parse(last_updated_at).iso8601,
+                oldest_created_at: Time.parse(from_datetime).iso8601,
               },
               batch_size: 100
             }
@@ -50,7 +50,7 @@ module Embulk
 
             proc = proc{ "" }
 
-            assert_nil(soap.each(last_updated_at, &proc))
+            assert_nil(soap.each(from_datetime, &proc))
           end
 
           class TestMetadata < self
@@ -65,14 +65,14 @@ module Embulk
               mock(@savon).call(:get_lead_changes, message: request) {
                 next_stream_activity_logs_response
               }
-              soap.metadata(last_updated_at)
+              soap.metadata(from_datetime)
             end
 
             def test_return_schema
               stub(@savon).call(:get_lead_changes, message: request) {
                 next_stream_activity_logs_response
               }
-              assert_equal(schema, soap.metadata(last_updated_at))
+              assert_equal(schema, soap.metadata(from_datetime))
             end
 
             private
@@ -80,7 +80,7 @@ module Embulk
             def request
               {
                 start_position: {
-                  oldest_created_at: Time.parse(last_updated_at).iso8601,
+                  oldest_created_at: Time.parse(from_datetime).iso8601,
                 },
                 batch_size: 100
               }
@@ -105,7 +105,7 @@ module Embulk
 
           private
 
-          def last_updated_at
+          def from_datetime
             "2015-07-06"
           end
 

data/test/override_assert_raise.rb

@@ -0,0 +1,18 @@
+module OverrideAssertRaise
+  # NOTE: Embulk 0.7.1+ required to raise ConfigError to do as `ConfigError.new("message")`,
+  #       original `assert_raise` method can't catch that, but `begin .. rescue` can.
+  #       So we override assert_raise as below.
+  def assert_raise(expected_class = StandardError, &block)
+    begin
+      block.call
+      assert_equal expected_class, nil
+    rescue ::Test::Unit::AssertionFailedError => e
+      # failed assert raises this Error and that extends StandardError, so rescue it first
+      raise e
+    rescue expected_class
+      assert true # passed
+    rescue => e
+      assert_equal(expected_class, e.class) # not expected one raised
+    end
+  end
+end

data/test/prepare_embulk.rb

@@ -1,10 +1,15 @@
 require "embulk/command/embulk_run"
 
-classpath_dir = Embulk.home("classpath")
-jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
-jars.each do |jar|
-  require File.join(classpath_dir, jar)
+if Embulk.respond_to?(:home)
+  # keep compatibility for Embulk 0.6.x
+  classpath_dir = Embulk.home("classpath")
+  jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
+  jars.each do |jar|
+    require File.join(classpath_dir, jar)
+  end
+  require "embulk/java/bootstrap"
+  require "embulk"
+else
+  require "embulk"
+  Embulk.setup
 end
-require "embulk/java/bootstrap"
-
-require "embulk"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - uu59
@@ -9,12 +9,12 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-30 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.11.1
   name: savon
@@ -22,13 +22,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.11.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: httpclient
@@ -36,16 +36,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.13
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: embulk
@@ -53,16 +53,16 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.6.13
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -70,13 +70,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -84,13 +84,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: pry
@@ -98,13 +98,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: test-unit
@@ -112,13 +112,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: test-unit-rr
@@ -126,13 +126,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: codeclimate-test-reporter
@@ -140,13 +140,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: everyleaf-embulk_helper
@@ -154,7 +154,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Loads records from Marketo.
@@ -165,15 +165,20 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .ruby-version
-- .travis.yml
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
+- ".travis.yml.erb"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - embulk-input-marketo.gemspec
+- gemfiles/embulk-0.6.0-latest
+- gemfiles/embulk-0.6.13
+- gemfiles/embulk-0.6.14
+- gemfiles/embulk-0.6.15
 - gemfiles/embulk-0.6.16
 - gemfiles/embulk-0.6.17
 - gemfiles/embulk-0.6.18
@@ -185,12 +190,19 @@ files:
 - gemfiles/embulk-0.6.24
 - gemfiles/embulk-0.6.25
 - gemfiles/embulk-0.6.26
+- gemfiles/embulk-0.6.27
+- gemfiles/embulk-0.7.0
+- gemfiles/embulk-0.7.0-latest
+- gemfiles/embulk-0.7.1
+- gemfiles/embulk-0.7.2
+- gemfiles/embulk-0.7.3
+- gemfiles/embulk-0.7.4
+- gemfiles/embulk-0.7.5
 - gemfiles/embulk-latest
 - gemfiles/template.erb
 - lib/embulk/input/marketo/activity_log.rb
 - lib/embulk/input/marketo/base.rb
 - lib/embulk/input/marketo/lead.rb
-- lib/embulk/input/marketo/timeslice.rb
 - lib/embulk/input/marketo_api.rb
 - lib/embulk/input/marketo_api/soap/activity_log.rb
 - lib/embulk/input/marketo_api/soap/base.rb
@@ -205,6 +217,7 @@ files:
 - test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
 - test/mute_logger.rb
+- test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
 homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -217,17 +230,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Marketo input plugin for Embulk
@@ -242,5 +255,6 @@ test_files:
 - test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
 - test/mute_logger.rb
+- test/override_assert_raise.rb
 - test/prepare_embulk.rb
 - test/run-test.rb

data/lib/embulk/input/marketo/timeslice.rb

@@ -1,120 +0,0 @@
-module Embulk
-  module Input
-    module Marketo
-      module Timeslice
-        TIMESLICE_COUNT_PER_TASK = 24
-
-        def self.included(klass)
-          klass.extend ClassMethods
-        end
-
-        module ClassMethods
-          def guess(config)
-            if config.param(:last_updated_at, :string, default: nil)
-              Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
-            end
-
-            client = soap_client(config)
-            metadata = client.metadata
-
-            return {"columns" => generate_columns(metadata)}
-          end
-
-          def transaction(config, &control)
-            endpoint_url = config.param(:endpoint, :string)
-
-            if config.param(:last_updated_at, :string, default: nil)
-              Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
-            end
-
-            from_datetime = config.param(:from_datetime, :string)
-            to_datetime = config.param(:to_datetime, :string, default: Time.now.to_s)
-
-            # check from/to format to parse
-            begin
-              Time.parse(from_datetime)
-              Time.parse(to_datetime)
-            rescue => e
-              # possibly Time.parse fail
-              raise ConfigError, e.message
-            end
-
-            if Time.parse(from_datetime) > Time.parse(to_datetime)
-              raise ConfigError, "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
-            end
-
-            ranges = timeslice(from_datetime, to_datetime, TIMESLICE_COUNT_PER_TASK)
-            task = {
-              endpoint_url: endpoint_url,
-              wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-              user_id: config.param(:user_id, :string),
-              encryption_key: config.param(:encryption_key, :string),
-              from_datetime: from_datetime,
-              to_datetime: to_datetime,
-              ranges: ranges,
-              columns: config.param(:columns, :array)
-            }
-
-            columns = []
-
-            task[:columns].each do |column|
-              name = column["name"]
-              type = column["type"].to_sym
-
-              columns << Column.new(nil, name, type, column["format"])
-            end
-
-            resume(task, columns, ranges.size, &control)
-          end
-
-          def timeslice(from, to, count)
-            generate_time_range(from, to).each_slice(count).to_a
-          end
-
-          def generate_time_range(from, to)
-            # e.g. from = 2010-01-01 15:00, to = 2010-01-03 09:30
-            # convert to such array:
-            # [
-            #   {from: 2010-01-01 15:00, to: 2010-01-01 16:00},
-            #   {from: 2010-01-01 16:00, to: 2010-01-01 17:00},
-            #   ...
-            #   {from: 2010-01-03 08:00, to: 2010-01-03 09:00},
-            #   {from: 2010-01-03 09:00, to: 2010-01-03 09:30},
-            # ]
-            # to fetch data from Marketo API with each day as
-            # desribed on official blog:
-            # http://developers.marketo.com/blog/performance-tuning-api-requests/
-            to ||= Time.now
-            from = Time.parse(from) unless from.is_a?(Time)
-            to = Time.parse(to) unless to.is_a?(Time)
-
-            result = []
-            since = from
-            while since < to
-              next_since = since + 3600
-              if to < next_since
-                next_since = to
-              end
-              result << {
-                "from" => since,
-                "to" => next_since
-              }
-              since = next_since
-            end
-            result
-          end
-
-          def resume(task, columns, count, &control)
-            commit_reports = yield(task, columns, count)
-
-            # When no task ran, commit_reports is empty
-            return {} if commit_reports.empty?
-            # all task returns same report as {from_datetime: to_datetime}
-            return commit_reports.first
-          end
-
-        end
-      end
-    end
-  end
-end