embulk-input-marketo 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ce46ab1c9fa86c191f9def74b1cc6ba7115dc48
-  data.tar.gz: 7f922b4f306b0d61b3cca0a805c25b3ea90e1ec0
+  metadata.gz: d4fb4473d0eb983aabb83a21397eb32dde08e678
+  data.tar.gz: 0c9fbac5a5cefcb755bc0a10be544d71a54eaba6
 SHA512:
-  metadata.gz: 32e9af2d5bbb8de6bca9b4fec3292d145e0aeb636961ccdea1298bf28e8a086da83c08b69ead0e86e978de6e334b247e65c20e9ac62353a12755d6a3c7cbba9e
-  data.tar.gz: 30a2f6cba66c48fc917ffe95abc504812ba17359b18ec498c94d125d6dc7a7b5990ccecf4dd1305a43f1fb631f1abca2464aeb448443f8e09ff209584b8db16f
+  metadata.gz: de9235beec534f01d4d54d4051b4f22678a4519b3a0c8cd929be61c3a981499238dc41b0cf0c2b979687e24713f717f9217e1f6b77c13b6d1eb80085b68e7215
+  data.tar.gz: 9516e1e4796f5711bae5747e7d3e636bc80542fb76d7f85d6b938bad091d294c6adb75043e7b0ef99e0cabe4351393e4f9f662980bf398ac575691818124c77f

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.4 - 2016-10-26
+* [enhancement] Validate wsdl_url and endpoint_url are the valid form [#61](https://github.com/treasure-data/embulk-input-marketo/pull/61)
+* [enhancement] Minor readme change [#58](https://github.com/treasure-data/embulk-input-marketo/pull/58)
+* [fixed] Fix error when retrying on guess/preview [#59](https://github.com/treasure-data/embulk-input-marketo/pull/59)
+* [enhancement] Try newer date at first on preview to avoid miss hit [#60](https://github.com/treasure-data/embulk-input-marketo/pull/60)
+
 ## 0.5.3 - 2016-07-01
 
 * [enhancement] make concurrent limit exceeded error retryable [#56](https://github.com/treasure-data/embulk-input-marketo/pull/56)

data/README.md

@@ -48,7 +48,7 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 ### marketo/activity_log
 
 - **endpoint** SOAP endpoint URL for your account (string, required)
-- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
+- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")(Note "?WSDL" needs to be in all caps)
 - **user_id** Your user id (string, reqiured)
 - **encryption_key** Your encryption key (string, reqiured)
 - **from_datetime** Fetch activity_logs since this time (string, required)

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.5.3"
+  spec.version       = "0.5.4"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."

data/lib/embulk/input/marketo/activity_log.rb

@@ -37,6 +37,8 @@ module Embulk
             retry_limit: config.param(:retry_limit, :integer, default: 5),
             columns: config.param(:columns, :array)
           }
+          validate_url(task[:endpoint_url], "endpoint")
+          validate_url(task[:wsdl_url], "wsdl")
 
           resume(task, embulk_columns(config), 1, &control)
         end

data/lib/embulk/input/marketo/base.rb

@@ -20,6 +20,8 @@ module Embulk
             user_id: config.param(:user_id, :string),
             encryption_key: config.param(:encryption_key, :string),
           }
+          validate_url(soap_config[:endpoint_url], "endpoint")
+          validate_url(soap_config[:wsdl_url], "wsdl")
 
           MarketoApi.soap_client(soap_config, target)
         end
@@ -99,6 +101,12 @@ module Embulk
 
         private
 
+        def self.validate_url(url, key)
+          URI.parse(url)
+        rescue URI::InvalidURIError
+          raise ConfigError.new("#{key}: '#{url}' is not a valid URL.")
+        end
+
         def preview?
           begin
             org.embulk.spi.Exec.isPreview()

data/lib/embulk/input/marketo/lead.rb

@@ -50,6 +50,8 @@ module Embulk
             append_processed_time_column: append_processed_time_column,
             columns: config.param(:columns, :array),
           }
+          validate_url(task[:endpoint_url], "endpoint")
+          validate_url(task[:wsdl_url], "wsdl")
 
           columns = embulk_columns(config)
           if append_processed_time_column
@@ -91,7 +93,12 @@ module Embulk
 
         def init
           @columns = task[:columns]
-          @ranges = task[:ranges][index]
+          if preview?
+            # Try newer date at first to reduce cache miss hit
+            @ranges = task[:ranges].flatten.sort_by{|range| Time.parse(range["to"])}.reverse
+          else
+            @ranges = task[:ranges][index]
+          end
           @soap = MarketoApi.soap_client(task, target)
           @append_processed_time_column = task[:append_processed_time_column]
           @options = {

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -43,9 +43,12 @@ module Embulk
           end
 
           def retryer(retry_options)
+            # guess/preview doesn't pass retry_options
+            initial_wait = retry_options[:retry_initial_wait_sec] || 1
+            retry_limit = retry_options[:retry_limit] || 5
             PerfectRetry.new do |config|
-              config.sleep = proc{|n| retry_options[:retry_initial_wait_sec] * (2 ** (n - 1))}
-              config.limit = retry_options[:retry_limit]
+              config.sleep = proc{|n| initial_wait * (2 ** (n - 1))}
+              config.limit = retry_limit
               config.dont_rescues = [Embulk::ConfigError]
               config.rescues = [StandardError, Timeout::Error]
               config.logger = Embulk.logger

data/test/embulk/input/marketo/test_activity_log.rb

@@ -32,6 +32,34 @@ module Embulk
             assert_equal(next_config_diff, actual)
           end
 
+          data do
+            [
+              ["endpoint", "endpoint"],
+              ["wsdl", "wsdl"],
+            ]
+          end
+          def test_invalid_url(key)
+            control = proc {} # dummy
+
+            settings = {
+              endpoint: "https://marketo.example.com",
+              wsdl: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+              columns: [
+                {"name" => "Name", "type" => "string"},
+              ],
+              from_datetime: Time.now,
+              to_datetime: Time.now + 3600,
+            }
+            settings[key.to_sym] = " invalid url "
+            config = DataSource[settings.to_a]
+
+            assert_raise(ConfigError) do
+              ActivityLog.transaction(config, &control)
+            end
+          end
+
           private
 
           def settings
@@ -105,6 +133,19 @@ module Embulk
             )
           end
 
+          data do
+            [
+              ["endpoint", "endpoint"],
+              ["wsdl", "wsdl"],
+            ]
+          end
+          def test_invalid_url(key)
+            config[key] = " invalid url "
+            assert_raise(ConfigError) do
+              Marketo::ActivityLog.guess(config)
+            end
+          end
+
           private
 
           def records

data/test/embulk/input/marketo/test_lead.rb

@@ -51,6 +51,34 @@ module Embulk
           end
         end
 
+        data do
+          [
+            ["endpoint", "endpoint"],
+            ["wsdl", "wsdl"],
+          ]
+        end
+        def test_invalid_url(key)
+          control = proc {} # dummy
+
+          settings = {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ],
+            from_datetime: Time.now,
+            to_datetime: Time.now + 3600,
+          }
+          settings[key.to_sym] = " invalid url "
+          config = DataSource[settings.to_a]
+
+          assert_raise(ConfigError) do
+            Lead.transaction(config, &control)
+          end
+        end
+
         def test_invalid_datetime_given
           control = proc {} # dummy
 
@@ -396,6 +424,19 @@ module Embulk
               Lead.guess(config)
             )
           end
+
+          data do
+            [
+              ["endpoint", "endpoint"],
+              ["wsdl", "wsdl"],
+            ]
+          end
+          def test_invalid_url(key)
+            config[key] = " invalid url "
+            assert_raise(ConfigError) do
+              Lead.guess(config)
+            end
+          end
         end
 
         def test_generate_columns

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-01 00:00:00.000000000 Z
+date: 2016-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement