embulk-input-marketo-through-proxy 0.6.20

data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java

@@ -0,0 +1,325 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin.PluginTask;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin.QueryBy;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class ProgramInputPluginTest
+{
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ExpectedException thrown = ExpectedException.none();
+
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public TestRule chain = RuleChain.outerRule(runtime).around(thrown);
+
+    private ConfigSource baseConfig;
+
+    private ProgramInputPlugin mockPlugin;
+
+    private MarketoRestClient mockRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        mockPlugin = Mockito.spy(new ProgramInputPlugin());
+        baseConfig = config();
+        mockRestClient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockRestClient).when(mockPlugin).createMarketoRestClient(Mockito.any(ProgramInputPlugin.PluginTask.class));
+    }
+
+//    -----------Verify configs --------------
+    @Test
+    public void testQueryByTagTypeConfigMissingTagType()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("tag_type and tag_value are required when query by Tag Type");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.TAG_TYPE)).set("tag_value", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByTagTypeConfigMissingTagValue()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("tag_type and tag_value are required when query by Tag Type");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.TAG_TYPE)).set("tag_type", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingEarliestUpatedAt()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`earliest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.DATE_RANGE)).set("latest_updated_at", Optional.of(Date.from(OffsetDateTime.now(ZoneOffset.UTC).minusDays(10).toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingLatestUpdatedAt()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`latest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.DATE_RANGE)).set("earliest_updated_at", Optional.of(Date.from(OffsetDateTime.now(ZoneOffset.UTC).minusDays(10).toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingLatestUpdatedAtNonIncremental()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`latest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("incremental", Boolean.FALSE)
+                        .set("earliest_updated_at", Optional.of(Date.from(OffsetDateTime.now(ZoneOffset.UTC).minusDays(10).toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testNoErrorQueryByDateRangeConfigHasReportDurationNonIncremental()
+    {
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("report_duration", Optional.of(60L * 1000))
+                        .set("incremental", Boolean.FALSE)
+                        .set("earliest_updated_at", Optional.of(Date.from(OffsetDateTime.now(ZoneOffset.UTC).minusDays(10).toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testNoErrorQueryByDateRangeConfigHasReportDuration()
+    {
+        ConfigSource config = baseConfig
+                        .set("report_duration", Optional.of(60L * 1000))
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(OffsetDateTime.now(ZoneOffset.UTC).minusDays(10).toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigHasEarliestUpdatedAtExceededNow()
+    {
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).plusDays(1);
+        OffsetDateTime latestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(2);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage(String.format("`earliest_updated_at` (%s) cannot precede the current date ", earliestUpdatedAt.format(DATE_FORMATTER)));
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigHasEarliestUpdatedAtExceededLatestUpdatedAt()
+    {
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(10);
+        OffsetDateTime latestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(20);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage(String.format("Invalid date range. `earliest_updated_at` (%s) cannot precede the `latest_updated_at` (%s).",
+                        earliestUpdatedAt.format(DATE_FORMATTER),
+                        latestUpdatedAt.format(DATE_FORMATTER)));
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testHasFilterTypeButMissingFilterValue()
+    {
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(20);
+        OffsetDateTime latestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(10);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("filter_value is required when selected filter_type");
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())))
+                        .set("filter_type", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testSkipIncrementalRunIfLastUpdatedAtExceedsNow()
+    {
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(20);
+        OffsetDateTime latestUpdatedAt = earliestUpdatedAt.plusDays(21);
+        //21 days
+        long reportDuration = 21 * 24 * 60 * 60 * 1000;
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())))
+                        .set("report_duration", reportDuration)
+                        .set("incremental", true);
+        ServiceResponseMapper<? extends ValueLocator> mapper = mockPlugin.buildServiceResponseMapper(config.loadConfig(PluginTask.class));
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        TaskReport taskReport = mockPlugin.ingestServiceData(config.loadConfig(PluginTask.class), recordImporter, 1, mockPageBuilder);
+        // page builder object should never get called.
+        Mockito.verify(mockPageBuilder, Mockito.never()).addRecord();
+
+        String earliestUpdatedAtStr = taskReport.get(String.class, "earliest_updated_at");
+        long duration = taskReport.get(Long.class, "report_duration");
+        assertEquals(duration, reportDuration);
+        assertEquals(earliestUpdatedAtStr, earliestUpdatedAt.format(DATE_FORMATTER));
+    }
+
+    @Test
+    public void testBuildConfigDiff()
+    {
+        TaskReport taskReport1 = Mockito.mock(TaskReport.class);
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(20);
+        OffsetDateTime latestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(10);
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())))
+                        .set("incremental", true);
+
+        ConfigDiff diff = mockPlugin.buildConfigDiff(config.loadConfig(PluginTask.class), Mockito.mock(Schema.class), 1, Arrays.asList(taskReport1));
+
+        long reportDuration = diff.get(Long.class, "report_duration");
+        String nextErliestUpdatedAt = diff.get(String.class, "earliest_updated_at");
+
+        assertEquals(reportDuration, Duration.between(earliestUpdatedAt, latestUpdatedAt).toMillis());
+        assertEquals(nextErliestUpdatedAt, latestUpdatedAt.plusSeconds(1).format(DATE_FORMATTER));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void testRun(ConfigSource config, Predicate<MarketoRestClient> expectedCall) throws IOException
+    {
+        // Mock response data
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/all_program_full.json"), javaType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+        Mockito.when(mockRestClient.getProgramsByTag(Mockito.anyString(), Mockito.anyString())).thenReturn(mockRecordPagingIterable);
+        Mockito.when(mockRestClient.getPrograms()).thenReturn(mockRecordPagingIterable);
+        Mockito.when(mockRestClient.getProgramsByDateRange(
+                      Mockito.any(Date.class),
+                      Mockito.any(Date.class),
+                      Mockito.nullable(String.class),
+                      Mockito.nullable(List.class))).thenReturn(mockRecordPagingIterable);
+
+        ServiceResponseMapper<? extends ValueLocator> mapper = mockPlugin.buildServiceResponseMapper(config.loadConfig(PluginTask.class));
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        mockPlugin.ingestServiceData(config.loadConfig(PluginTask.class), recordImporter, 1, mockPageBuilder);
+
+        // The method getProgramByTag is called 1 time
+//        Mockito.verify(mockRestClient, Mockito.times(1)).getProgramsByTag(Mockito.anyString(), Mockito.anyString());
+        expectedCall.apply(mockRestClient);
+
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        // 17 columns
+        assertEquals(embulkSchema.size(), 17);
+        // verify 3 times the method setLong for column id has been called
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Mockito.verify(mockPageBuilder, Mockito.times(3)).setLong(Mockito.eq(embulkSchema.lookupColumn("id")), longArgumentCaptor.capture());
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        assertArrayEquals(new Long[]{1004L, 1001L, 1003L}, allValues.toArray());
+    }
+
+    @Test
+    public void testRunQueryByTagType() throws IOException
+    {
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.TAG_TYPE))
+                        .set("tag_type", Optional.of("dummy"))
+                        .set("tag_value", Optional.of("dummy"));
+        Predicate<MarketoRestClient> expectedCall = mockRest -> {
+            Mockito.verify(mockRest, Mockito.times(1)).getProgramsByTag(Mockito.anyString(), Mockito.anyString());
+            return true;
+        };
+        testRun(config, expectedCall);
+    }
+
+    @Test
+    public void testRunWithoutQueryBy() throws IOException
+    {
+        Predicate<MarketoRestClient> expectedCall = input -> {
+            Mockito.verify(input, Mockito.times(1)).getPrograms();
+            return true;
+        };
+        testRun(baseConfig, expectedCall);
+    }
+
+    @Test
+    public void testRunQueryByDateRange() throws IOException
+    {
+        OffsetDateTime earliestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(20);
+        OffsetDateTime latestUpdatedAt = OffsetDateTime.now(ZoneOffset.UTC).minusDays(10);
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(Date.from(earliestUpdatedAt.toInstant())))
+                        .set("latest_updated_at", Optional.of(Date.from(latestUpdatedAt.toInstant())));
+        Predicate<MarketoRestClient> expectedCall = input -> {
+            Mockito.verify(input, Mockito.times(1)).getProgramsByDateRange(
+                            Mockito.any(Date.class),
+                            Mockito.any(Date.class),
+                            Mockito.nullable(String.class),
+                            Mockito.nullable(List.class));
+            return true;
+        };
+        testRun(config, expectedCall);
+    }
+
+    public ConfigSource config() throws IOException
+    {
+        ConfigLoader configLoader = runtime.getInjector().getInstance(ConfigLoader.class);
+        return configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
+    }
+}

data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java

@@ -0,0 +1,368 @@
+package org.embulk.input.marketo.rest;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpResponseException;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.input.marketo.exception.MarketoAPIException;
+import org.embulk.input.marketo.model.MarketoError;
+import org.embulk.input.marketo.model.MarketoResponse;
+import org.embulk.spi.DataException;
+import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
+import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
+import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.function.ThrowingRunnable;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Created by tai.khuu on 9/21/17.
+ */
+public class MarketoBaseRestClientTest
+{
+    private static final String IDENTITY_END_POINT = "identityEndPoint";
+
+    private static final int MARKETO_LIMIT_INTERVAL_MILIS = 1000;
+
+    private MarketoBaseRestClient marketoBaseRestClient;
+
+    private Jetty92RetryHelper mockJetty92;
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    @Before
+    public void prepare()
+    {
+        mockJetty92 = Mockito.mock(Jetty92RetryHelper.class);
+        marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", Optional.<String>absent(), MARKETO_LIMIT_INTERVAL_MILIS, 60000, mockJetty92);
+    }
+
+    @Test
+    public void testGetAccessToken()
+    {
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class))).thenReturn("{\n" +
+                "    \"access_token\": \"access_token\",\n" +
+                "    \"token_type\": \"bearer\",\n" +
+                "    \"expires_in\": 3599,\n" +
+                "    \"scope\": \"tai@treasure-data.com\"\n" +
+                "}");
+        String accessToken = marketoBaseRestClient.getAccessToken();
+        Assert.assertEquals("access_token", accessToken);
+    }
+
+    @Test
+    public void testGetAccessTokenRequester()
+    {
+        ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn("{\"access_token\": \"access_token\"}");
+        String accessToken = marketoBaseRestClient.getAccessToken();
+        Assert.assertEquals("access_token", accessToken);
+        Jetty92SingleRequester jetty92SingleRequester = jetty92SingleRequesterArgumentCaptor.getValue();
+        HttpClient client = Mockito.mock(HttpClient.class);
+        Response.Listener listener = Mockito.mock(Response.Listener.class);
+        Request mockRequest = Mockito.mock(Request.class);
+        Mockito.when(client.newRequest(Mockito.eq(IDENTITY_END_POINT + MarketoRESTEndpoint.ACCESS_TOKEN.getEndpoint()))).thenReturn(mockRequest);
+        Request request1 = Mockito.mock(Request.class);
+        Mockito.when(mockRequest.method(Mockito.eq(HttpMethod.GET))).thenReturn(request1);
+        jetty92SingleRequester.requestOnce(client, listener);
+        Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("client_id"), Mockito.eq("clientId"));
+        Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("client_secret"), Mockito.eq("clientSecret"));
+        Mockito.verify(request1, Mockito.times(1)).param(Mockito.eq("grant_type"), Mockito.eq("client_credentials"));
+
+        // By default the partner id is not set
+        Mockito.verify(request1, Mockito.never()).param(Mockito.eq("partner_id"), Mockito.anyString());
+
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createHttpResponseException(502)));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
+        // Retry SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
+        // When EOFException is wrapped in IOException it should be retried too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
+        // Retry TimeoutException when it is wrapped in IOException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
+    }
+
+    @Test
+    public void testGetAccessTokenRequestShouldHavePartnerId()
+    {
+        final String partnerId = "sample_partner_id";
+        mockJetty92 = Mockito.mock(Jetty92RetryHelper.class);
+        ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn("{\"access_token\": \"access_token\"}");
+
+        MarketoBaseRestClient restClient = Mockito.spy(new MarketoBaseRestClient("identityEndPoint",
+                "clientId",
+                "clientSecret",
+                Optional.of(partnerId),
+                MARKETO_LIMIT_INTERVAL_MILIS,
+                60000,
+                mockJetty92));
+
+        // call method for evaluation
+        restClient.getAccessToken();
+
+        Jetty92SingleRequester singleRequester = jetty92SingleRequesterArgumentCaptor.getValue();
+
+        HttpClient client = Mockito.mock(HttpClient.class);
+        Request request = Mockito.mock(Request.class);
+
+        Request mockRequest = Mockito.mock(Request.class);
+        Mockito.when(client.newRequest(Mockito.eq(IDENTITY_END_POINT + MarketoRESTEndpoint.ACCESS_TOKEN.getEndpoint()))).thenReturn(mockRequest);
+        Mockito.when(mockRequest.method(Mockito.eq(HttpMethod.GET))).thenReturn(request);
+        singleRequester.requestOnce(client, Mockito.mock(Response.Listener.class));
+
+        Mockito.verify(request, Mockito.times(1)).param(Mockito.eq("client_id"), Mockito.eq("clientId"));
+        Mockito.verify(request, Mockito.times(1)).param(Mockito.eq("client_secret"), Mockito.eq("clientSecret"));
+        Mockito.verify(request, Mockito.times(1)).param(Mockito.eq("grant_type"), Mockito.eq("client_credentials"));
+        Mockito.verify(request, Mockito.times(1)).param(Mockito.eq("partner_id"), Mockito.eq(partnerId));
+    }
+
+    @Test
+    public void testGetAccessTokenWithError()
+    {
+        ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn("{\n" +
+                "    \"error\": \"invalid_client\",\n" +
+                "    \"error_description\": \"Bad client credentials\"\n" +
+                "}");
+        try {
+            marketoBaseRestClient.getAccessToken();
+        }
+        catch (DataException ex) {
+            Assert.assertEquals("Bad client credentials", ex.getMessage());
+            return;
+        }
+        Assert.fail();
+    }
+
+    @Test
+    public void testGetAccessTokenThrowHttpResponseException() throws Exception
+    {
+        HttpClient client = Mockito.mock(HttpClient.class);
+
+        Jetty92ClientCreator clientCreator = Mockito.mock(Jetty92ClientCreator.class);
+        Mockito.doReturn(client).when(clientCreator).createAndStart();
+
+        Request request = Mockito.mock(Request.class);
+        Mockito.doReturn(request).when(client).newRequest(Mockito.anyString());
+        Mockito.doReturn(request).when(request).method(HttpMethod.GET);
+
+        HttpResponseException exception = new HttpResponseException("{\"error\":\"invalid_client\",\"error_description\":\"Bad client credentials\"}", Mockito.mock(Response.class));
+        Mockito.doThrow(exception).when(request).send(Mockito.any(Response.Listener.class));
+
+        Jetty92RetryHelper retryHelper = new Jetty92RetryHelper(1, 1, 1, clientCreator);
+        final MarketoBaseRestClient restClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", Optional.<String>absent(), MARKETO_LIMIT_INTERVAL_MILIS, 1000, retryHelper);
+
+        // calling method should wrap the HttpResponseException by ConfigException
+        Assert.assertThrows(ConfigException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                restClient.getAccessToken();
+            }
+        });
+    }
+
+    @Test
+    public void tetDoGetThrowHttpResponseException() throws Exception
+    {
+        final MarketoBaseRestClient client = doRequestWithWrapper(HttpMethod.GET);
+        // calling method should wrap the HttpResponseException by DataException
+        Assert.assertThrows(DataException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                client.doGet("test_target", null, null, new MarketoResponseJetty92EntityReader<String>(1000));
+            }
+        });
+    }
+
+    @Test
+    public void tetDoPostThrowHttpResponseException() throws Exception
+    {
+        final MarketoBaseRestClient client = doRequestWithWrapper(HttpMethod.POST);
+        // calling method should wrap the HttpResponseException by DataException
+        Assert.assertThrows(DataException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                client.doPost("test_target", null, null, "{\"any\": \"any\"}", new MarketoResponseJetty92EntityReader<String>(1000));
+            }
+        });
+    }
+
+    private MarketoBaseRestClient doRequestWithWrapper(HttpMethod method) throws Exception
+    {
+        HttpClient client = Mockito.mock(HttpClient.class);
+
+        Jetty92ClientCreator clientCreator = Mockito.mock(Jetty92ClientCreator.class);
+        Mockito.doReturn(client).when(clientCreator).createAndStart();
+
+        Request request = Mockito.mock(Request.class);
+        Mockito.doReturn(request).when(client).newRequest(Mockito.anyString());
+        Mockito.doReturn(request).when(request).method(method);
+
+        HttpResponseException exception = new HttpResponseException("{\"error\":\"1035\",\"error_description\":\"Unsupported filter type for target subscription: updatedAt\"}", Mockito.mock(Response.class));
+        Mockito.doThrow(exception).when(request).send(Mockito.any(Response.Listener.class));
+
+        Jetty92RetryHelper retryHelper = new Jetty92RetryHelper(1, 1, 1, clientCreator);
+        final MarketoBaseRestClient restClient = Mockito.spy(new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", Optional.<String>absent(), MARKETO_LIMIT_INTERVAL_MILIS, 1000, retryHelper));
+        Mockito.doReturn("test_access_token").when(restClient).getAccessToken();
+
+        return restClient;
+    }
+
+    @Test
+    public void testDoPost() throws Exception
+    {
+        MarketoBaseRestClient spy = Mockito.spy(marketoBaseRestClient);
+        spy.doPost("target", Maps.<String, String>newHashMap(), new ImmutableListMultimap.Builder<String, String>().build(), "test_content", new StringJetty92ResponseEntityReader(10));
+        Mockito.verify(spy, Mockito.times(1)).doRequest(Mockito.anyString(), Mockito.eq(HttpMethod.POST), Mockito.any(Map.class), Mockito.any(Multimap.class), Mockito.any(StringContentProvider.class), Mockito.any(StringJetty92ResponseEntityReader.class));
+    }
+
+    @Test
+    public void testDoGet() throws Exception
+    {
+        MarketoBaseRestClient spy = Mockito.spy(marketoBaseRestClient);
+        spy.doGet("target", Maps.<String, String>newHashMap(), new ImmutableListMultimap.Builder<String, String>().build(), new StringJetty92ResponseEntityReader(10));
+        Mockito.verify(spy, Mockito.times(1)).doRequest(Mockito.anyString(), Mockito.eq(HttpMethod.GET), Mockito.any(Map.class), Mockito.any(Multimap.class), Mockito.isNull(ContentProvider.class), Mockito.any(StringJetty92ResponseEntityReader.class));
+    }
+
+    @Test
+    public void testDoRequestRequester() throws Exception
+    {
+        MarketoBaseRestClient spy = Mockito.spy(marketoBaseRestClient);
+        StringContentProvider contentProvider = new StringContentProvider("Content", StandardCharsets.UTF_8);
+        ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
+
+        MarketoResponse<Object> expectedMarketoResponse = new MarketoResponse<>();
+
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(MarketoResponseJetty92EntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn(expectedMarketoResponse);
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class))).thenReturn("{\"access_token\": \"access_token\"}");
+
+        String target = "target";
+        HashMap<String, String> headers = Maps.<String, String>newHashMap();
+        headers.put("testHeader1", "testHeaderValue1");
+
+        ImmutableListMultimap<String, String> build = new ImmutableListMultimap.Builder<String, String>().put("param", "param1").build();
+
+        MarketoResponse<Object> marketoResponse = spy.doRequest(target, HttpMethod.POST, headers, build, contentProvider, new MarketoResponseJetty92EntityReader<Object>(10));
+
+        HttpClient client = Mockito.mock(HttpClient.class);
+        Response.Listener listener = Mockito.mock(Response.Listener.class);
+        Request mockRequest = Mockito.mock(Request.class);
+        Mockito.when(client.newRequest(Mockito.eq(target))).thenReturn(mockRequest);
+
+        Mockito.when(mockRequest.method(Mockito.eq(HttpMethod.POST))).thenReturn(mockRequest);
+        Jetty92SingleRequester jetty92SingleRequester = jetty92SingleRequesterArgumentCaptor.getValue();
+        jetty92SingleRequester.requestOnce(client, listener);
+
+        Assert.assertEquals(expectedMarketoResponse, marketoResponse);
+
+        Mockito.verify(mockRequest, Mockito.times(1)).header(Mockito.eq("testHeader1"), Mockito.eq("testHeaderValue1"));
+        Mockito.verify(mockRequest, Mockito.times(1)).header(Mockito.eq("Authorization"), Mockito.eq("Bearer access_token"));
+        Mockito.verify(mockRequest, Mockito.times(1)).param(Mockito.eq("param"), Mockito.eq("param1"));
+        Mockito.verify(mockRequest, Mockito.times(1)).content(Mockito.eq(contentProvider));
+    }
+
+    @Test
+    public void testDoRequesterRetry() throws Exception
+    {
+        MarketoBaseRestClient spy = Mockito.spy(marketoBaseRestClient);
+        ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
+
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(MarketoResponseJetty92EntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn(new MarketoResponse<>());
+        Mockito.when(mockJetty92.requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class))).thenReturn("{\"access_token\": \"access_token\"}");
+
+        spy.doRequest("", HttpMethod.POST, null, null, null, new MarketoResponseJetty92EntityReader<Object>(10));
+
+        HttpClient client = Mockito.mock(HttpClient.class);
+        Response.Listener listener = Mockito.mock(Response.Listener.class);
+        Request mockRequest = Mockito.mock(Request.class);
+        Mockito.when(client.newRequest(Mockito.anyString())).thenReturn(mockRequest);
+
+        Mockito.when(mockRequest.method(Mockito.eq(HttpMethod.POST))).thenReturn(mockRequest);
+
+        Jetty92SingleRequester jetty92SingleRequester = jetty92SingleRequesterArgumentCaptor.getValue();
+        jetty92SingleRequester.requestOnce(client, listener);
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createHttpResponseException(502)));
+
+        Assert.assertFalse(jetty92SingleRequester.toRetry(createHttpResponseException(400)));
+
+        Assert.assertFalse(jetty92SingleRequester.toRetry(createMarketoAPIException("ERR", "ERR")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("606", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("615", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("602", "")));
+        // Should retry 601 error too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("601", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("604", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("608", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("611", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("615", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("713", "")));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("1029", "")));
+        // Retry wrap SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
+        // When EOFException is wrapped in IOException it should be retried too
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
+        // Retry TimeoutException when it is wrapped in IOException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
+
+        // Retry SocketTimeoutException, TimeoutException and EOFException
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
+        Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
+       // Call 3 times First call then 602 error and  601 error
+        Mockito.verify(mockJetty92, Mockito.times(3)).requestWithRetry(Mockito.any(StringJetty92ResponseEntityReader.class), Mockito.any(Jetty92SingleRequester.class));
+    }
+
+    private HttpResponseException createHttpResponseException(int statusCode)
+    {
+        HttpResponseException exception = Mockito.mock(HttpResponseException.class);
+        Response response = Mockito.mock(Response.class);
+        Mockito.when(exception.getResponse()).thenReturn(response);
+        Mockito.when(response.getStatus()).thenReturn(statusCode);
+        return exception;
+    }
+
+    private MarketoAPIException createMarketoAPIException(String code, String error)
+    {
+        MarketoError marketoError = new MarketoError();
+        marketoError.setCode(code);
+        marketoError.setMessage(error);
+        return new MarketoAPIException(Lists.newArrayList(marketoError));
+    }
+}