embulk-input-marketo 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dde3369b27304f507037e63efd59b2e4e7bdc449
-  data.tar.gz: fd6df82180b3d3fbf3ef0835b16ea6894f1e4161
+  metadata.gz: 8e89a971b0dca06b010f1081c37969c59c1ef04c
+  data.tar.gz: 8dda51b8e935c023e514c6db6e1e15b868fc59ff
 SHA512:
-  metadata.gz: 3ec3f1ca521a50e53e6c62fe7c51e67c44c09bad74936ff4e34b52fe6e00f721848bef35a35d2030c75a641d405126ea7efe28871928fc9319041b5dcc200f13
-  data.tar.gz: 303b8b3f47e439d4e863f6f157f64de2cc7d46fd46b2487f206573e7e4d0a43e00e6edd6a902b10e10200204e2fd1149e0982352fda66efbbd2500909959363c
+  metadata.gz: a4f0072db71cdbd8ea2b6cad7c1071c5fd5ca05efe827d27d0e6185142028464890f98b413343e98396330d037ced52e114bbeec9c229a2746f573bfda070b7a
+  data.tar.gz: e7e7f52566640fba0a3d5cb23087dc10f6e94dd7635e6a17e36018cd8aed257ced0608a50793596608e84b0f5cb31e2bbd6c36506185b1f98bb83bd7f4004f7f

data/.travis.yml

@@ -5,87 +5,41 @@ addons:
     repo_token:
       secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
 
-notifications:
-  slack:
-    secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
-
 before_install:
   - |
     # Currently, Travis can't treat jruby 9.0.1.0
-    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
-      rvm get head
-      rvm use jruby-9.0.1.0 --install
-      ruby -v
-    fi
+    ruby -v
+    rvm get head
+    rvm use jruby-9.0.5.0 --install
+    ruby -v
+    gem i bundler
+    bundle install
 
 jdk: oraclejdk8
 
 rvm:
-  - jruby-19mode
-  - jruby-9.0.0.0
+  - jruby-9.0.5.0
+
+script: bundle exec rake test
 
 gemfile:
-  - gemfiles/embulk-0.6.0-latest
-  - gemfiles/embulk-0.6.13
-  - gemfiles/embulk-0.6.14
-  - gemfiles/embulk-0.6.15
-  - gemfiles/embulk-0.6.16
-  - gemfiles/embulk-0.6.17
-  - gemfiles/embulk-0.6.18
-  - gemfiles/embulk-0.6.19
-  - gemfiles/embulk-0.6.20
-  - gemfiles/embulk-0.6.21
-  - gemfiles/embulk-0.6.22
-  - gemfiles/embulk-0.6.23
-  - gemfiles/embulk-0.6.24
-  - gemfiles/embulk-0.6.25
-  - gemfiles/embulk-0.6.26
-  - gemfiles/embulk-0.6.27
-  - gemfiles/embulk-0.7.0
-  - gemfiles/embulk-0.7.0-latest
-  - gemfiles/embulk-0.7.1
-  - gemfiles/embulk-0.7.2
-  - gemfiles/embulk-0.7.3
-  - gemfiles/embulk-0.7.4
-  - gemfiles/embulk-0.7.5
+  - gemfiles/embulk-0.8.0-latest
+  - gemfiles/embulk-0.8.7
+  - gemfiles/embulk-0.8.8
   - gemfiles/embulk-latest
 
 matrix:
   exclude:
     - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
   include:
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.0-latest}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.13}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.14}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.15}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.16}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.17}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.18}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.19}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.20}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.21}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.22}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.23}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.24}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.25}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.26}
-    - {rvm: jruby-19mode, gemfile: gemfiles/embulk-0.6.27}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.0-latest}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.1}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.2}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.3}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.4}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.7.5}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.0-latest}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.7}
+    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.8}
     - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-latest}
     
 
   allow_failures:
-    - gemfile: gemfiles/embulk-0.6.22
-    - gemfile: gemfiles/embulk-0.7.0
-    - gemfile: gemfiles/embulk-0.7.1
     # Ignore failure for *-latest
-    - gemfile: embulk-0.6.0-latest
-    - gemfile: embulk-0.7.0-latest
+    - gemfile: embulk-0.8.0-latest
     - gemfile: embulk-latest
     

data/.travis.yml.erb

@@ -5,24 +5,22 @@ addons:
     repo_token:
       secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
 
-notifications:
-  slack:
-    secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
-
 before_install:
   - |
     # Currently, Travis can't treat jruby 9.0.1.0
-    if [[ ${TRAVIS_RUBY_VERSION} = "jruby-9.0.0.0" ]];then
-      rvm get head
-      rvm use jruby-9.0.1.0 --install
-      ruby -v
-    fi
+    ruby -v
+    rvm get head
+    rvm use jruby-9.0.5.0 --install
+    ruby -v
+    gem i bundler
+    bundle install
 
 jdk: oraclejdk8
 
 rvm:
-  - jruby-19mode
-  - jruby-9.0.0.0
+  - jruby-9.0.5.0
+
+script: bundle exec rake test
 
 gemfile:
 <% versions.each do |file| -%>
@@ -38,9 +36,6 @@ matrix:
     <% end %>
 
   allow_failures:
-    - gemfile: gemfiles/embulk-0.6.22
-    - gemfile: gemfiles/embulk-0.7.0
-    - gemfile: gemfiles/embulk-0.7.1
     # Ignore failure for *-latest
     <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
 - gemfile: <%= file %>

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.5.0 - 2016-04-06
+
+This version drops old Embulk supports. Embulk 0.8 or later is required since this version.
+
+* [enhancement] Add tests for Embulk 0.8 and drop support old Embulk [#52](https://github.com/treasure-data/embulk-input-marketo/pull/52)
+* [maintenance] Refactor lead and activity [#51](https://github.com/treasure-data/embulk-input-marketo/pull/51)
+* [maintenance] Refactor retry [#50](https://github.com/treasure-data/embulk-input-marketo/pull/50)
+
 ## 0.4.0 - 2015-10-30
 
 This version drops scheduled execution with marketo/lead.

data/README.md

@@ -14,10 +14,10 @@ This plugin uses Marketo SOAP API.
 
 ## Overview
 
-Required Embulk version >= 0.6.13.
+Required Embulk version >= 0.8.7 (since 0.5.0).
 
 * **Plugin type**: input
-* **Resume supported**: yes for `marketo/lead`, no for `marketo/activity_log`
+* **Resume supported**: no for `marketo/lead`, yes for `marketo/activity_log`
 * **Cleanup supported**: no
 * **Guess supported**: yes
 
@@ -35,8 +35,6 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 
 ### marketo/lead
 
-**NOTE: If you use feature of scheduled execution (resume) with marketo/lead, you should not specify `to_datetime` because this plugin can't place next to_datetime (can't know the date to run with new config).**
-
 - **endpoint** SOAP endpoint URL for your account (string, required)
 - **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
 - **user_id** Your user id (string, reqiured)

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.4.0"
+  spec.version       = "0.5.0"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'savon', ['~> 2.11.1']
   spec.add_dependency 'httpclient'
+  spec.add_dependency 'perfect_retry', ["~> 0.3.2"]
   spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/gemfiles/embulk-latest

@@ -1,4 +1,4 @@
 source 'https://rubygems.org/'
 gemspec :path => '../'
 
-gem "embulk", "> 0.6.13"
+gem "embulk", "> 0.8.7"

data/lib/embulk/input/marketo/activity_log.rb

@@ -58,25 +58,18 @@ module Embulk
         def init
           @columns = task[:columns]
           @soap = MarketoApi.soap_client(task, target)
-        end
-
-        def run
-          options = {
+          @options = {
             retry_initial_wait_sec: task[:retry_initial_wait_sec],
             retry_limit: task[:retry_limit],
             to: task[:to_datetime],
             batch_size: (preview? ? PREVIEW_COUNT : BATCH_SIZE_DEFAULT),
           }
+        end
 
+        def run
           counter = 0
-          latest_updated_at = @soap.each(task[:from_datetime], options) do |activity_log|
-            values = @columns.map do |column|
-              name = column["name"].to_s
-              value = activity_log[name]
-              cast_value(column, value)
-            end
-
-            page_builder.add(values)
+          latest_updated_at = @soap.each(task[:from_datetime], @options) do |activity_log|
+            page_builder.add(format_record(activity_log))
             break if preview? && (counter += 1) >= PREVIEW_COUNT
           end
 
@@ -89,6 +82,14 @@ module Embulk
 
           return task_report
         end
+
+        def format_record(activity_log)
+          @columns.map do |column|
+            name = column["name"].to_s
+            value = activity_log[name]
+            cast_value(column, value)
+          end
+        end
       end
     end
   end

data/lib/embulk/input/marketo/lead.rb

@@ -94,30 +94,19 @@ module Embulk
           @ranges = task[:ranges][index]
           @soap = MarketoApi.soap_client(task, target)
           @append_processed_time_column = task[:append_processed_time_column]
-        end
-
-        def run
-          options = {
+          @options = {
             retry_initial_wait_sec: task[:retry_initial_wait_sec],
             retry_limit: task[:retry_limit],
           }
-          options[:batch_size] = PREVIEW_COUNT if preview?
+          @options[:batch_size] = PREVIEW_COUNT if preview?
+        end
 
+        def run
           counter = 0
           catch(:finish) do
             @ranges.each do |range|
-              soap.each(range, options) do |lead|
-                values = @columns.map do |column|
-                  name = column["name"].to_s
-                  value = (lead[name] || {})[:value]
-                  cast_value(column, value)
-                end
-
-                if @append_processed_time_column
-                  values << Time.parse(range["from"])
-                end
-
-                page_builder.add(values)
+              soap.each(range, @options) do |lead|
+                page_builder.add(format_record(lead, range))
                 throw(:finish) if preview? && (counter += 1) >= PREVIEW_COUNT
               end
             end
@@ -128,6 +117,19 @@ module Embulk
           task_report = {}
           return task_report
         end
+
+        def format_record(lead, range)
+          values = @columns.map do |column|
+            name = column["name"].to_s
+            value = (lead[name] || {})[:value]
+            cast_value(column, value)
+          end
+
+          if @append_processed_time_column
+            values << Time.parse(range["from"])
+          end
+          values
+        end
       end
     end
   end

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -57,42 +57,44 @@ module Embulk
 
           def fetch(request, options={}, &block)
             response = savon_call(:get_lead_changes, {message: request}, options)
-            remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
+            remaining = response.xpath('//remainingCount').text.to_i
             Embulk.logger.info "Remaining records: #{remaining}"
 
-            activities_list = response.body[:success_get_lead_changes][:result][:lead_change_record_list]
+            activities = response.xpath('//leadChangeRecord')
 
-            if activities_list.nil?
+            if activities.empty?
               Embulk.logger.info "No record is fetched."
               return {remaining_count: 0, offset: nil, from_datetime: nil}
             end
 
-            activities = activities_list[:lead_change_record].sort_by { |activity| Time.parse(activity[:activity_date_time]) }
-
             activities.each do |activity|
-              record = {
-                "id" => activity[:id],
-                "activity_date_time" => activity[:activity_date_time],
-                "activity_type" => activity[:activity_type],
-                "mktg_asset_name" => activity[:mktg_asset_name],
-                "mkt_person_id" => activity[:mkt_person_id],
-              }
-
-              activity[:activity_attributes][:attribute].each do |attributes|
-                name = attributes[:attr_name]
-                value = attributes[:attr_value]
-
-                record[name] = value
-              end
-
-              block.call(record)
+              process_record(activity, &block)
             end
 
             {
               remaining_count: remaining,
-              offset: response.body[:success_get_lead_changes][:result][:new_start_position][:offset],
-              from_datetime: activities.last[:activity_date_time]
+              offset: response.xpath('//newStartPosition/offset').text,
+              from_datetime: activities.map{|a| Time.parse(a.at('./activityDateTime').text) }.max,
+            }
+          end
+
+          def process_record(activity, &block)
+            record = {
+              "id" => activity.at("./id").text,
+              "activity_date_time" => activity.at('./activityDateTime').text,
+              "activity_type" => activity.at('./activityType').text,
+              "mktg_asset_name" => activity.at('./mktgAssetName').text,
+              "mkt_person_id" => activity.at('./mktPersonId').text,
             }
+
+            activity.xpath('./activityAttributes/attribute').each do |attr|
+              name = attr.xpath('attrName').text
+              value = attr.xpath('attrValue').text
+
+              record[name] = value
+            end
+
+            block.call(record)
           end
         end
       end

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -1,5 +1,6 @@
 require "savon"
 require "httpclient" # net/http can't verify cert correctly
+require "perfect_retry"
 
 module Embulk
   module Input
@@ -41,8 +42,19 @@ module Embulk
             )
           end
 
+          def retryer(retry_options)
+            PerfectRetry.new do |config|
+              config.sleep = proc{|n| retry_options[:retry_initial_wait_sec] * (2 ** (n - 1))}
+              config.limit = retry_options[:retry_limit]
+              config.dont_rescues = [Embulk::ConfigError]
+              config.rescues = [StandardError, Timeout::Error]
+              config.logger = Embulk.logger
+              config.log_level = nil
+            end
+          end
+
           def savon_call(operation, locals={}, retry_options={})
-            with_retry(retry_options) do
+            retryer(retry_options).with_retry do
               catch_unretryable_error do
                 savon.call(operation, locals.merge(advanced_typecasting: false))
               end
@@ -60,23 +72,6 @@ module Embulk
             }
           end
 
-          def with_retry(options, &block)
-            wait_sec = options[:retry_initial_wait_sec]
-            count = 0
-            begin
-              yield
-            rescue Embulk::ConfigError => e # TODO: Add Embulk::DataError for Embulk 0.7+
-              raise e
-            rescue ::Timeout::Error, StandardError => e
-              count += 1
-              raise e if count > options[:retry_limit]
-              Embulk.logger.warn "Retrying after #{wait_sec} seconds [#{count}/#{RETRY_TIMEOUT_COUNT}] Error: #{e}"
-              sleep wait_sec
-              wait_sec *= 2
-              retry
-            end
-          end
-
           def catch_unretryable_error(&block)
             yield
           rescue Savon::SOAPFault => e
@@ -101,7 +96,7 @@ module Embulk
               # unretryable error such as Authentication Failed, Invalid Request, etc.
               raise ConfigError.new soap_message
             end
-          rescue SocketError, Errno::ECONNREFUSED => e
+          rescue SocketError, ::Java::JavaNet::UnknownHostException, Errno::ECONNREFUSED => e
             # maybe endpoint/wsdl domain was wrong
             Embulk.logger.debug "Connection error: endpoint=#{endpoint} wsdl=#{wsdl}"
             raise ConfigError.new "Connection error: #{e.message} (endpoint is '#{endpoint}')"