embulk-input-marketo 0.6.7 → 0.6.8.alpha1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af6c681344d0710a3803c198298732c5fe271140
-  data.tar.gz: 3816444d0f15f51aab2efffd6f1c6637c09833a1
+  metadata.gz: d23d2cfa89b67071f75d5972006bfc13723f4879
+  data.tar.gz: a23b0cc19c7e884d9e75126319e718244e7e7905
 SHA512:
-  metadata.gz: 37719bf91a9f4a1ba5456770cb7a6ec92036e6854bfa02224ee87ad4d853dfd57e6d541a2e002f4832c0865ebaa26c000b745d0612357cc64d9f59a7c83630bf
-  data.tar.gz: 911428dc09e0b2a61ffd708bc3ab613f106b4dd53ca54f99a293d9a1c9c28c42061949b3c97c4697da3a4de02b23954d4dcafcacc3286fe4c6280c55c0433b7e
+  metadata.gz: 266e3cbbf8e3ee5bbe289fdb4b4327b6b6da510ca9bd09514f0e2b698ed25462fa13e297741f46804cd478eaf9f36cc09bd38a7de62ab17ced20ba4af7006a22
+  data.tar.gz: d65566dbe869de9925df1055acfc7dfe7997e170222737e3f5953c77020f6e409d99a5ea5be727837878372d5b65f7be47c676b291266990aff92a271d8e8943

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.7 - 2018-02-26
+* [fixed] Remove de-duplication logic [#83](https://github.com/treasure-data/embulk-input-marketo/pull/83)
+
 ## 0.6.6 - 2018-01-30
 * [fixed] Fix JettyRetryHelper not closed [#82](https://github.com/treasure-data/embulk-input-marketo/pull/82)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.7"
+version = "0.6.8.alpha1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -43,12 +43,13 @@ checkstyle {
     toolVersion = '6.14.1'
 }
 checkstyleMain {
+    exclude '**/**/CsvTokenizer*'
     configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-    ignoreFailures = true
+    ignoreFailures = false
 }
 checkstyleTest {
     configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-    ignoreFailures = true
+    ignoreFailures = false
 }
 task checkstyle(type: Checkstyle) {
     classpath = sourceSets.main.output + sourceSets.test.output

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -112,6 +112,15 @@ public class MarketoBaseRestClient implements AutoCloseable
             {
                 return response.getStatus() == 502;
             }
+
+            @Override
+            protected boolean isExceptionToRetry(Exception exception)
+            {
+                if (exception instanceof ExecutionException) {
+                    return this.toRetry((Exception) exception.getCause());
+                }
+                return exception instanceof TimeoutException || exception instanceof SocketTimeoutException || exception instanceof EOFException || super.isExceptionToRetry(exception);
+            }
         });
 
         MarketoAccessTokenResponse accessTokenResponse;
@@ -189,11 +198,7 @@ public class MarketoBaseRestClient implements AutoCloseable
             protected boolean isExceptionToRetry(Exception exception)
             {
                 if (exception instanceof ExecutionException) {
-                    this.toRetry((Exception) exception.getCause());
-                }
-                //Anything that is EOFException or cause by EOF exception
-                if (exception instanceof EOFException || exception.getCause() instanceof EOFException) {
-                    return true;
+                    return this.toRetry((Exception) exception.getCause());
                 }
                 if (exception instanceof MarketoAPIException) {
                     //Retry Authenticate Exception
@@ -201,6 +206,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                     String code = error.getCode();
                     switch (code) {
                         case "602":
+                        case "601":
                             LOGGER.info("Access token expired");
                             renewAccessToken();
                             return true;
@@ -219,7 +225,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                             return false;
                     }
                 }
-                return exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception);
+                return exception instanceof EOFException || exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception);
             }
         });
     }

data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java

@@ -26,9 +26,13 @@ import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
+import java.io.EOFException;
+import java.net.SocketTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
 
 /**
  * Created by tai.khuu on 9/21/17.
@@ -72,18 +76,25 @@ public class MarketoBaseRestClientTest
         Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn("{\"access_token\": \"access_token\"}");
         String accessToken = marketoBaseRestClient.getAccessToken();
         Assert.assertEquals("access_token", accessToken);
-        Jetty92SingleRequester value = jetty92SingleRequesterArgumentCaptor.getValue();
+        Jetty92SingleRequester jetty92SingleRequester = jetty92SingleRequesterArgumentCaptor.getValue();
         HttpClient client = Mockito.mock(HttpClient.class);
         Response.Listener listener = Mockito.mock(Response.Listener.class);
         Request mockRequest = Mockito.mock(Request.class);
         Mockito.when(client.newRequest(Mockito.eq(IDENTITY_END_POINT + MarketoRESTEndpoint.ACCESS_TOKEN.getEndpoint()))).thenReturn(mockRequest);
         Request request1 = Mockito.mock(Request.class);
         Mockito.when(mockRequest.method(Mockito.eq(HttpMethod.GET))).thenReturn(request1);
-        value.requestOnce(client, listener);
+        jetty92SingleRequester.requestOnce(client, listener);
         Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("client_id"), Mockito.eq("clientId"));
         Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("client_secret"), Mockito.eq("clientSecret"));
         Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("grant_type"), Mockito.eq("client_credentials"));
-        Assert.assertTrue(value.toRetry(createHttpResponseException(502)));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createHttpResponseException(502)));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
+        // Retry SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
     }
     @Test
     public void testGetAccessTokenWithError()
@@ -184,8 +195,18 @@ public class MarketoBaseRestClientTest
         Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("606", "")));
         Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("615", "")));
         Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("602", "")));
-
-        Mockito.verify(mockJetty92, Mockito.times(2)).requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class));
+        // Should retry 601 error too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("601", "")));
+        // Retry wrap SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
+        // Retry SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
+       // Call 3 times First call then 602 error and  601 error
+        Mockito.verify(mockJetty92, Mockito.times(3)).requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class));
     }
 
     private HttpResponseException createHttpResponseException(int statusCode)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8.alpha1
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-26 00:00:00.000000000 Z
+date: 2018-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,6 @@ files:
 - src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java
 - src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java
 - src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java
-- src/test/java/org/embulk/input/marketo/TestMarketoInputPlugin.java
 - src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java
@@ -125,9 +124,9 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-input-marketo-0.6.8.alpha1.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.7.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -145,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9

data/src/test/java/org/embulk/input/marketo/TestMarketoInputPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.input.marketo;
-
-public class TestMarketoInputPlugin
-{
-}