embulk-input-marketo 0.6.8 → 0.6.9.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2094efbeffeb797a5bc4c07124b38c8390371a6b
-  data.tar.gz: b147ee805bb3160efd15364d0514f37b9e76a09e
+  metadata.gz: f075ab4c25711804d71689227cd164392154cb09
+  data.tar.gz: a565d6381ac3ac89b21397c6b522c4c3192c6f60
 SHA512:
-  metadata.gz: a5c0f3a0905f53adba639d790db5bd0e125aed74c151186f95090438ccfa1b2344243fbfbe98d91fa59b3b80d0f352016ed887ed278068e09070457544e79f9b
-  data.tar.gz: 3f21bfeaac29db7a019f6cbc40c4a0875d65c83805afc2813573aee5bbaa6dca7954411ab5a801650b98f6dc4b1868b9316199070a5e29b239c7f02d53248364
+  metadata.gz: 7593bdf9bacc02e3e13fc28009c12ef68b69790022d66613fa58fd27227f7616a8a4c4e15843d75e474fc84922ae53ce1e6ae63dd8363d45553f16a09484f0c0
+  data.tar.gz: 7307a87215da07e29703d537603d6b49fdfaf985fb61f30998f4153ab7ce8a14d21144ffc9c0ce8fe7492edcb1e04a14b7cd5a6f4702b94a666b9ac92b045b1a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.8 - 2018-04-12
+* [fixed] Fix incorrect incorrect retry logic [#84](https://github.com/treasure-data/embulk-input-marketo/pull/84)
+
 ## 0.6.7 - 2018-02-26
 * [fixed] Remove de-duplication logic [#83](https://github.com/treasure-data/embulk-input-marketo/pull/83)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.8"
+version = "0.6.9.alpha"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -30,18 +30,6 @@ public class MarketoInputPluginDelegate
         @Config("target")
         Target getTarget();
 
-        @Config("maximum_retries")
-        @ConfigDefault("3")
-        Integer getMaximumRetries();
-
-        @Config("initial_retry_interval_milis")
-        @ConfigDefault("20000")
-        Integer getInitialRetryIntervalMilis();
-
-        @Config("maximum_retries_interval_milis")
-        @ConfigDefault("120000")
-        Integer getMaximumRetriesIntervalMilis();
-
         //We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
         //We will hide the real from_date, and set it when validating task
         @Config("hidden_from_date")

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -36,7 +36,7 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
             extends RestClientInputTaskBase, MarketoRestClient.PluginTask
     {
         @Config("maximum_retries")
-        @ConfigDefault("3")
+        @ConfigDefault("7")
         Integer getMaximumRetries();
 
         @Config("initial_retry_interval_milis")

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -33,7 +33,7 @@ import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTR
 import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
 
 /**
- * Marketo base rest client that provide
+ * Marketo base rest client
  * Created by tai.khuu on 9/7/17.
  */
 public class MarketoBaseRestClient implements AutoCloseable
@@ -44,8 +44,6 @@ public class MarketoBaseRestClient implements AutoCloseable
 
     private static final String AUTHORIZATION_HEADER = "Authorization";
 
-    protected static final long READ_TIMEOUT_MILLIS = 30000;
-
     private String identityEndPoint;
 
     private String clientId;
@@ -54,19 +52,22 @@ public class MarketoBaseRestClient implements AutoCloseable
 
     private String accessToken;
 
-    private int marketoLimitIntervalMilis;
+    private int marketoLimitIntervalMillis;
 
     private Jetty92RetryHelper retryHelper;
 
+    protected long readTimeoutMillis;
+
     protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false).configure(ALLOW_UNQUOTED_CONTROL_CHARS, false);
 
-    MarketoBaseRestClient(String identityEndPoint, String clientId, String clientSecret, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
+    MarketoBaseRestClient(String identityEndPoint, String clientId, String clientSecret, int marketoLimitIntervalMillis, long readTimeoutMillis, Jetty92RetryHelper retryHelper)
     {
         this.identityEndPoint = identityEndPoint;
         this.clientId = clientId;
         this.clientSecret = clientSecret;
+        this.readTimeoutMillis = readTimeoutMillis;
         this.retryHelper = retryHelper;
-        this.marketoLimitIntervalMilis = marketoLimitIntervalMilis;
+        this.marketoLimitIntervalMillis = marketoLimitIntervalMillis;
     }
 
     private void renewAccessToken()
@@ -93,7 +94,7 @@ public class MarketoBaseRestClient implements AutoCloseable
         params.put("client_id", clientId);
         params.put("client_secret", clientSecret);
         params.put("grant_type", "client_credentials");
-        String response = retryHelper.requestWithRetry(new StringJetty92ResponseEntityReader(READ_TIMEOUT_MILLIS), new Jetty92SingleRequester()
+        String response = retryHelper.requestWithRetry(new StringJetty92ResponseEntityReader(readTimeoutMillis), new Jetty92SingleRequester()
         {
             @Override
             public void requestOnce(HttpClient client, Response.Listener responseListener)
@@ -116,10 +117,14 @@ public class MarketoBaseRestClient implements AutoCloseable
             @Override
             protected boolean isExceptionToRetry(Exception exception)
             {
-                if (exception instanceof ExecutionException) {
+                if (exception instanceof TimeoutException || exception instanceof SocketTimeoutException || exception instanceof EOFException || super.isExceptionToRetry(exception)) {
+                    return true;
+                }
+                // unwrap
+                if (exception instanceof ExecutionException || (exception instanceof IOException && exception.getCause() != null)) {
                     return this.toRetry((Exception) exception.getCause());
                 }
-                return exception instanceof TimeoutException || exception instanceof SocketTimeoutException || exception instanceof EOFException || super.isExceptionToRetry(exception);
+                return false;
             }
         });
 
@@ -197,7 +202,10 @@ public class MarketoBaseRestClient implements AutoCloseable
             @Override
             protected boolean isExceptionToRetry(Exception exception)
             {
-                if (exception instanceof ExecutionException) {
+                if (exception instanceof EOFException || exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception)) {
+                    return true;
+                }
+                if (exception instanceof ExecutionException || (exception instanceof IOException && exception.getCause() != null)) {
                     return this.toRetry((Exception) exception.getCause());
                 }
                 if (exception instanceof MarketoAPIException) {
@@ -212,7 +220,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                             return true;
                         case "606":
                             try {
-                                Thread.sleep(marketoLimitIntervalMilis);
+                                Thread.sleep(marketoLimitIntervalMillis);
                             }
                             catch (InterruptedException e) {
                                 LOGGER.error("Encounter exception when waiting for interval limit", e);
@@ -225,7 +233,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                             return false;
                     }
                 }
-                return exception instanceof EOFException || exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception);
+                return false;
             }
         });
     }

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -104,16 +104,20 @@ public class MarketoRestClient extends MarketoBaseRestClient
         @ConfigDefault("200")
         Integer getMaxReturn();
         void setMaxReturn(Integer maxReturn);
+
+        @Config("read_timeout_millis")
+        @ConfigDefault("60000")
+        Long getReadTimeoutMillis();
     }
 
     public MarketoRestClient(PluginTask task, Jetty92RetryHelper retryHelper)
     {
-        this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getMarketoLimitIntervalMilis(), retryHelper);
+        this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getReadTimeoutMillis(), task.getMarketoLimitIntervalMilis(), retryHelper);
     }
 
-    public MarketoRestClient(String endPoint, String identityEndPoint, String clientId, String clientSecret, Integer batchSize, Integer maxReturn, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
+    public MarketoRestClient(String endPoint, String identityEndPoint, String clientId, String clientSecret, Integer batchSize, Integer maxReturn, long readTimeoutMilis, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
     {
-        super(identityEndPoint, clientId, clientSecret, marketoLimitIntervalMilis, retryHelper);
+        super(identityEndPoint, clientId, clientSecret, marketoLimitIntervalMilis, readTimeoutMilis, retryHelper);
         this.endPoint = endPoint;
         this.batchSize = batchSize;
         this.maxReturn = maxReturn;
@@ -121,7 +125,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
 
     public List<MarketoField> describeLead()
     {
-        MarketoResponse<ObjectNode> jsonResponse = doGet(endPoint + MarketoRESTEndpoint.DESCRIBE_LEAD.getEndpoint(), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(READ_TIMEOUT_MILLIS));
+        MarketoResponse<ObjectNode> jsonResponse = doGet(endPoint + MarketoRESTEndpoint.DESCRIBE_LEAD.getEndpoint(), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(this.readTimeoutMillis));
         List<MarketoField> marketoFields = new ArrayList<>();
         List<ObjectNode> fields = jsonResponse.getResult();
         for (int i = 0; i < fields.size(); i++) {
@@ -175,7 +179,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
         MarketoResponse<ObjectNode> marketoResponse = null;
         try {
             LOGGER.info("Send bulk extract request [{}]", request);
-            marketoResponse = doPost(endPoint + endpoint.getEndpoint(), null, null, OBJECT_MAPPER.writeValueAsString(request), new MarketoResponseJetty92EntityReader<ObjectNode>(READ_TIMEOUT_MILLIS));
+            marketoResponse = doPost(endPoint + endpoint.getEndpoint(), null, null, OBJECT_MAPPER.writeValueAsString(request), new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
         }
         catch (JsonProcessingException e) {
             LOGGER.error("Encounter exception when deserialize bulk extract request", e);
@@ -203,7 +207,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
     {
         MarketoResponse<ObjectNode> marketoResponse = doPost(endPoint + marketoRESTEndpoint.getEndpoint(
                 new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, null,
-                new MarketoResponseJetty92EntityReader<ObjectNode>(READ_TIMEOUT_MILLIS));
+                new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
         if (!marketoResponse.isSuccess()) {
             MarketoError error = marketoResponse.getErrors().get(0);
             throw new DataException(String.format("Can't start job for export Job id : %s, error code: %s, error message: %s", exportId, error.getCode(), error.getMessage()));
@@ -245,7 +249,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
         long now = System.currentTimeMillis();
         while (true) {
             MarketoResponse<ObjectNode> marketoResponse = doGet(this.endPoint + marketoRESTEndpoint.getEndpoint(
-                    new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(READ_TIMEOUT_MILLIS));
+                    new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
             if (marketoResponse.isSuccess()) {
                 ObjectNode objectNode = marketoResponse.getResult().get(0);
                 String status = objectNode.get("status").asText();
@@ -290,7 +294,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
             headers.put(RANGE_HEADER, bulkExtractRangeHeader.toRangeHeaderValue());
             LOGGER.info("Range header value [{}]", bulkExtractRangeHeader.toRangeHeaderValue());
         }
-        return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), headers, null, new MarketoInputStreamResponseEntityReader(READ_TIMEOUT_MILLIS));
+        return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), headers, null, new MarketoInputStreamResponseEntityReader(readTimeoutMillis));
     }
 
     public RecordPagingIterable<ObjectNode> getLists()
@@ -345,7 +349,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 if (parameters != null) {
                     params.putAll(parameters);
                 }
-                MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(READ_TIMEOUT_MILLIS, recordClass));
+                MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(readTimeoutMillis, recordClass));
                 return new RecordPagingIterable.OffsetPage<>(marketoResponse.getResult(), offset + marketoResponse.getResult().size(), marketoResponse.getResult().size() == maxReturn);
             }
         });
@@ -386,7 +390,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 }
                 //Let do GET Disguise in POST here to overcome Marketo URI Too long error
                 FormContentProvider formContentProvider = new FormContentProvider(fields);
-                MarketoResponse<T> marketoResponse = doPost(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(READ_TIMEOUT_MILLIS, recordClass), formContentProvider);
+                MarketoResponse<T> marketoResponse = doPost(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(readTimeoutMillis, recordClass), formContentProvider);
                 return new RecordPagingIterable.TokenPage<>(marketoResponse.getResult(), marketoResponse.getNextPageToken(), marketoResponse.getNextPageToken() != null);
             }
         });

data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java

@@ -27,6 +27,7 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.EOFException;
+import java.io.IOException;
 import java.net.SocketTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
@@ -53,7 +54,7 @@ public class MarketoBaseRestClientTest
     public void prepare()
     {
         mockJetty92 = Mockito.mock(Jetty92RetryHelper.class);
-        marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, mockJetty92);
+        marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, 60000, mockJetty92);
     }
 
     @Test
@@ -95,6 +96,10 @@ public class MarketoBaseRestClientTest
         Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
         Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
         Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
+        // When EOFException is wrapped in IOException it should be retried too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
+        // Retry TimeoutException when it is wrapped in IOException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
     }
     @Test
     public void testGetAccessTokenWithError()
@@ -201,6 +206,11 @@ public class MarketoBaseRestClientTest
         Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
         Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
         Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
+        // When EOFException is wrapped in IOException it should be retried too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
+        // Retry TimeoutException when it is wrapped in IOException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
+
         // Retry SocketTimeoutException, TimeoutException and EOFException
         Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
         Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.6.9.alpha
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-02 00:00:00.000000000 Z
+date: 2018-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -123,10 +123,10 @@ files:
 - src/test/resources/fixtures/lists_response.json
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.8.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.9.alpha.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -144,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9