embulk-input-marketo 0.6.13 → 0.6.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 935d1b6ae8e0f35d81eaf082150dee811383a35d
-  data.tar.gz: 2284b4fdd5b7766cbbc4097f35b2c503ab4ee6b7
+  metadata.gz: a1ecb21401432881f5191869093f9e26d7eecd77
+  data.tar.gz: 80bce389f5f836747a900ce87d2d637125df6f0b
 SHA512:
-  metadata.gz: 27216b6dfc1e75a62fccc8d2266c3c267ddb85eedf2129c42e4361fde72849f8a0bd20432440e2bad549808cc27971a60747f1efcaf90aa711a13f71f9706039
-  data.tar.gz: 28b3f19096057d34c9cf091603f9d79e47424b30c9b4122a7929ff4ae4998c79adcd129817908290b9c327426c18815482d94a7f8c56cb885498a7992d230530
+  metadata.gz: 290ecc3ed678f765358615a0fb2e71bc61142999cf5cae9afa1c4aa60619df2e37778a884201f1505c478aef1d29311afd460f16d705785fc71a4f46f3f9c50c
+  data.tar.gz: 03e2c3625ade8587e3c1730f8ce3a6ffb3baa77195c7a26030fe988cdd22486150862e759c752f27c91ec58cdb20fb2b76441ff46f289936b38128d4b2de202c

data/.travis.yml

@@ -3,3 +3,4 @@ jdk:
 - oraclejdk8
 script:
 - ./gradlew --info check jacocoTestReport
+dist: trusty

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.6.15 - 2019-09-20
+* [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
+
+## 0.6.14 - 2019-08-19
+* [enhancement] Improve exception handling [#93](https://github.com/treasure-data/embulk-input-marketo/pull/93)
+
+## 0.6.13 - 2019-01-21
+* [enhance] Add more error code to retry [#91](https://github.com/treasure-data/embulk-input-marketo/pull/91)
+
 ## 0.6.12 - 2018-11-09
 * [enhance] Implement Custom Object [#90](https://github.com/treasure-data/embulk-input-marketo/pull/90)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.13"
+version = "0.6.15"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -42,8 +42,7 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
             return new FileInputStream(service.extractAllActivity(fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
-            LOGGER.error("Exception when trying to extract activity", e);
-            throw new DataException("Error when trying to extract activity");
+            throw new RuntimeException("Exception when trying to extract activity", e);
         }
     }
 

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java

@@ -15,6 +15,7 @@ import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
+
 import java.util.Iterator;
 
 public class CustomObjectInputPlugin extends MarketoBaseInputPluginDelegate<CustomObjectInputPlugin.PluginTask>
@@ -71,5 +72,4 @@ public class CustomObjectInputPlugin extends MarketoBaseInputPluginDelegate<Cust
             return customObjectResponseMapperBuilder.buildServiceResponseMapper(task);
         }
     }
-
 }

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java

@@ -12,6 +12,7 @@ import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -52,8 +52,7 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
             return new FileInputStream(service.extractLead(fromDate.toDate(), toDate.toDate(), fieldNames, task.getIncrementalColumn().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
-            LOGGER.error("File not found", e);
-            throw new DataException("Error when extract lead");
+            throw new RuntimeException("File not found", e);
         }
     }
 

data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java

@@ -179,7 +179,6 @@ public class ProgramInputPlugin extends MarketoBaseInputPluginDelegate<ProgramIn
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
         // set next next earliestUpdatedAt, latestUpdatedAt
         if (task.getQueryBy().isPresent() && task.getQueryBy().get() == QueryBy.DATE_RANGE && task.getIncremental()) {
-
             DateTime earliest = new DateTime(task.getEarliestUpdatedAt().get());
             DateTime latest = new DateTime(task.getLatestUpdatedAt().get());
 

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -5,11 +5,13 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpResponseException;
 import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.util.StringContentProvider;
 import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.config.ConfigException;
 import org.embulk.input.marketo.exception.MarketoAPIException;
 import org.embulk.input.marketo.model.MarketoAccessTokenResponse;
 import org.embulk.input.marketo.model.MarketoError;
@@ -72,7 +74,7 @@ public class MarketoBaseRestClient implements AutoCloseable
 
     private void renewAccessToken()
     {
-        accessToken = requestAccessToken();
+        accessToken = getAccessTokenWithWrappedException();
     }
 
     @VisibleForTesting
@@ -81,7 +83,7 @@ public class MarketoBaseRestClient implements AutoCloseable
         if (accessToken == null) {
             synchronized (this) {
                 if (accessToken == null) {
-                    accessToken = requestAccessToken();
+                    accessToken = getAccessTokenWithWrappedException();
                 }
             }
         }
@@ -146,7 +148,7 @@ public class MarketoBaseRestClient implements AutoCloseable
 
     protected <T> T doGet(final String target, final Map<String, String> headers, final Multimap<String, String> params, Jetty92ResponseReader<T> responseReader)
     {
-        return doRequest(target, HttpMethod.GET, headers, params, null, responseReader);
+        return doRequestWithWrappedException(target, HttpMethod.GET, headers, params, null, responseReader);
     }
 
     protected <T> T doPost(final String target, final Map<String, String> headers, final Multimap<String, String> params, final String content, Jetty92ResponseReader<T> responseReader)
@@ -160,7 +162,39 @@ public class MarketoBaseRestClient implements AutoCloseable
 
     protected <T> T doPost(final String target, final Map<String, String> headers, final Multimap<String, String> params, Jetty92ResponseReader<T> responseReader, final ContentProvider content)
     {
-        return doRequest(target, HttpMethod.POST, headers, params, content, responseReader);
+        return doRequestWithWrappedException(target, HttpMethod.POST, headers, params, content, responseReader);
+    }
+
+    private String getAccessTokenWithWrappedException()
+    {
+        try {
+            return requestAccessToken();
+        }
+        catch (Exception e) {
+            if (e instanceof HttpResponseException) {
+                throw new ConfigException(e.getMessage());
+            }
+            if (e.getCause() instanceof HttpResponseException) {
+                throw new ConfigException(e.getCause().getMessage());
+            }
+            throw e;
+        }
+    }
+
+    private <T> T doRequestWithWrappedException(final String target, final HttpMethod method, final Map<String, String> headers, final Multimap<String, String> params, final ContentProvider contentProvider, Jetty92ResponseReader<T> responseReader)
+    {
+        try {
+            return doRequest(target, method, headers, params, contentProvider, responseReader);
+        }
+        catch (Exception e) {
+            if (e instanceof MarketoAPIException || e instanceof HttpResponseException) {
+                throw new DataException(e.getMessage());
+            }
+            if (e.getCause() instanceof MarketoAPIException || e.getCause() instanceof HttpResponseException) {
+                throw new DataException(e.getCause().getMessage());
+            }
+            throw e;
+        }
     }
 
     protected <T> T doRequest(final String target, final HttpMethod method, final Map<String, String> headers, final Multimap<String, String> params, final ContentProvider contentProvider, Jetty92ResponseReader<T> responseReader)

data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java

@@ -21,6 +21,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
+
 import java.io.IOException;
 import java.util.List;
 

data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java

@@ -7,7 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Optional;
 import com.google.common.base.Predicate;
-
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.RecordImporter;
@@ -45,11 +45,9 @@ import java.util.List;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
 public class ProgramInputPluginTest
 {
-    private static final DateTimeFormatter DATE_FORMATER = DateTimeFormat.forPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
 
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
@@ -153,7 +151,7 @@ public class ProgramInputPluginTest
         DateTime earliestUpdatedAt = DateTime.now().plusDays(1);
         DateTime latestUpdatedAt = DateTime.now().minusDays(2);
         thrown.expect(ConfigException.class);
-        thrown.expectMessage(String.format("`earliest_updated_at` (%s) cannot precede the current date ", earliestUpdatedAt.toString(DATE_FORMATER)));
+        thrown.expectMessage(String.format("`earliest_updated_at` (%s) cannot precede the current date ", earliestUpdatedAt.toString(DATE_FORMATTER)));
         ConfigSource config = baseConfig
                         .set("query_by", Optional.of(QueryBy.DATE_RANGE))
                         .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
@@ -168,8 +166,8 @@ public class ProgramInputPluginTest
         DateTime latestUpdatedAt = DateTime.now().minusDays(20);
         thrown.expect(ConfigException.class);
         thrown.expectMessage(String.format("Invalid date range. `earliest_updated_at` (%s) cannot precede the `latest_updated_at` (%s).",
-                        earliestUpdatedAt.toString(DATE_FORMATER),
-                        latestUpdatedAt.toString(DATE_FORMATER)));
+                        earliestUpdatedAt.toString(DATE_FORMATTER),
+                        latestUpdatedAt.toString(DATE_FORMATTER)));
 
         ConfigSource config = baseConfig
                         .set("query_by", Optional.of(QueryBy.DATE_RANGE))
@@ -218,7 +216,7 @@ public class ProgramInputPluginTest
         String earliestUpdatedAtStr = taskReport.get(String.class, "earliest_updated_at");
         long duration = taskReport.get(Long.class, "report_duration");
         assertEquals(duration, reportDuration);
-        assertEquals(earliestUpdatedAtStr, earliestUpdatedAt.toString(DATE_FORMATER));
+        assertEquals(earliestUpdatedAtStr, earliestUpdatedAt.toString(DATE_FORMATTER));
     }
 
     @Test
@@ -239,7 +237,7 @@ public class ProgramInputPluginTest
         String nextErliestUpdatedAt = diff.get(String.class, "earliest_updated_at");
 
         assertEquals(reportDuration, new Duration(earliestUpdatedAt, latestUpdatedAt).getMillis());
-        assertEquals(nextErliestUpdatedAt, latestUpdatedAt.plusSeconds(1).toString(DATE_FORMATER));
+        assertEquals(nextErliestUpdatedAt, latestUpdatedAt.plusSeconds(1).toString(DATE_FORMATTER));
     }
 
     @SuppressWarnings("unchecked")

data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java

@@ -12,10 +12,12 @@ import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.util.StringContentProvider;
 import org.eclipse.jetty.http.HttpMethod;
 import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
 import org.embulk.input.marketo.exception.MarketoAPIException;
 import org.embulk.input.marketo.model.MarketoError;
 import org.embulk.input.marketo.model.MarketoResponse;
 import org.embulk.spi.DataException;
+import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
 import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
@@ -23,6 +25,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.function.ThrowingRunnable;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
@@ -119,6 +122,86 @@ public class MarketoBaseRestClientTest
         Assert.fail();
     }
 
+    @Test
+    public void testGetAccessTokenThrowHttpResponseException() throws Exception
+    {
+        HttpClient client = Mockito.mock(HttpClient.class);
+
+        Jetty92ClientCreator clientCreator = Mockito.mock(Jetty92ClientCreator.class);
+        Mockito.doReturn(client).when(clientCreator).createAndStart();
+
+        Request request = Mockito.mock(Request.class);
+        Mockito.doReturn(request).when(client).newRequest(Mockito.anyString());
+        Mockito.doReturn(request).when(request).method(HttpMethod.GET);
+
+        HttpResponseException exception = new HttpResponseException("{\"error\":\"invalid_client\",\"error_description\":\"Bad client credentials\"}", Mockito.mock(Response.class));
+        Mockito.doThrow(exception).when(request).send(Mockito.any(Response.Listener.class));
+
+        Jetty92RetryHelper retryHelper = new Jetty92RetryHelper(1, 1, 1, clientCreator);
+        final MarketoBaseRestClient restClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, 1000, retryHelper);
+
+        // calling method should wrap the HttpResponseException by ConfigException
+        Assert.assertThrows(ConfigException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                restClient.getAccessToken();
+            }
+        });
+    }
+
+    @Test
+    public void tetDoGetThrowHttpResponseException() throws Exception
+    {
+        final MarketoBaseRestClient client = doRequestWithWrapper(HttpMethod.GET);
+        // calling method should wrap the HttpResponseException by DataException
+        Assert.assertThrows(DataException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                client.doGet("test_target", null, null, new MarketoResponseJetty92EntityReader<String>(1000));
+            }
+        });
+    }
+
+    @Test
+    public void tetDoPostThrowHttpResponseException() throws Exception
+    {
+        final MarketoBaseRestClient client = doRequestWithWrapper(HttpMethod.POST);
+        // calling method should wrap the HttpResponseException by DataException
+        Assert.assertThrows(DataException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                client.doPost("test_target", null, null, "{\"any\": \"any\"}", new MarketoResponseJetty92EntityReader<String>(1000));
+            }
+        });
+    }
+
+    private MarketoBaseRestClient doRequestWithWrapper(HttpMethod method) throws Exception
+    {
+        HttpClient client = Mockito.mock(HttpClient.class);
+
+        Jetty92ClientCreator clientCreator = Mockito.mock(Jetty92ClientCreator.class);
+        Mockito.doReturn(client).when(clientCreator).createAndStart();
+
+        Request request = Mockito.mock(Request.class);
+        Mockito.doReturn(request).when(client).newRequest(Mockito.anyString());
+        Mockito.doReturn(request).when(request).method(method);
+
+        HttpResponseException exception = new HttpResponseException("{\"error\":\"1035\",\"error_description\":\"Unsupported filter type for target subscription: updatedAt\"}", Mockito.mock(Response.class));
+        Mockito.doThrow(exception).when(request).send(Mockito.any(Response.Listener.class));
+
+        Jetty92RetryHelper retryHelper = new Jetty92RetryHelper(1, 1, 1, clientCreator);
+        final MarketoBaseRestClient restClient = Mockito.spy(new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, 1000, retryHelper));
+        Mockito.doReturn("test_access_token").when(restClient).getAccessToken();
+
+        return restClient;
+    }
+
     @Test
     public void testDoPost() throws Exception
     {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.13
+  version: 0.6.15
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-21 00:00:00.000000000 Z
+date: 2019-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -137,9 +137,9 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.15.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.13.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo