embulk-input-marketo 0.5.7.alpha.6 → 0.6.0.alpha.1

data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java

@@ -0,0 +1,94 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+/**
+ * Created by khuutantaitai on 10/3/17.
+ */
+public class LeadBulkExtractInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private LeadBulkExtractInputPlugin bulkExtractInputPlugin;
+
+    private ConfigSource configSource;
+
+    private MarketoRestClient mockMarketoRestclient;
+
+    @Before
+    public void prepare() throws IOException
+    {
+        bulkExtractInputPlugin = spy(new LeadBulkExtractInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/lead_bulk_extract_config.yaml"));
+        mockMarketoRestclient = mock(MarketoRestClient.class);
+        doReturn(mockMarketoRestclient).when(bulkExtractInputPlugin).createMarketoRestClient(any(LeadBulkExtractInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testRun() throws InterruptedException, IOException
+    {
+        LeadBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(LeadBulkExtractInputPlugin.PluginTask.class);
+        DateTime startDate = new DateTime(task.getFromDate());
+        PageBuilder pageBuilder = mock(PageBuilder.class);
+        String exportId1 = "exportId1";
+        String exportId2 = "exportId2";
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), javaType);
+        List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
+        when(mockMarketoRestclient.describeLead()).thenReturn(marketoFields);
+        when(mockMarketoRestclient.createLeadBulkExtract(any(Date.class), any(Date.class), any(List.class), any(String.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
+        when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId1))).thenReturn(this.getClass().getResourceAsStream("/fixtures/lead_extract1.csv"));
+        when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId2))).thenReturn(this.getClass().getResourceAsStream("/fixtures/leads_extract2.csv"));
+        ServiceResponseMapper<? extends ValueLocator> mapper = bulkExtractInputPlugin.buildServiceResponseMapper(task);
+        bulkExtractInputPlugin.validateInputTask(task);
+        TaskReport taskReport = bulkExtractInputPlugin.ingestServiceData(task, mapper.createRecordImporter(), 1, pageBuilder);
+        ArgumentCaptor<Long> argumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Column idColumn = mapper.getEmbulkSchema().lookupColumn("mk_id");
+        verify(pageBuilder, times(19)).setLong(eq(idColumn), argumentCaptor.capture());
+        verify(mockMarketoRestclient, times(1)).startLeadBulkExtract(eq(exportId1));
+        verify(mockMarketoRestclient, times(1)).waitLeadExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
+        verify(mockMarketoRestclient, times(1)).startLeadBulkExtract(eq(exportId2));
+        verify(mockMarketoRestclient, times(1)).waitLeadExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
+        String filterField = "updatedAt";
+        verify(mockMarketoRestclient, times(1)).createLeadBulkExtract(startDate.toDate(), startDate.plusDays(30).toDate(), fieldNameFromMarketoFields, filterField);
+        DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
+        verify(mockMarketoRestclient, times(1)).createLeadBulkExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate(), fieldNameFromMarketoFields, filterField);
+        List<Long> leadIds = argumentCaptor.getAllValues();
+        assertEquals(19, leadIds.size());
+        long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
+        assertTrue(taskReport.get(Set.class, "latest_uids").isEmpty());
+        assertEquals(1504888753000L, latestFetchTime);
+        assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java

@@ -0,0 +1,99 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+/**
+ * Created by tai.khuu on 10/10/17.
+ */
+public class LeadWithListInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private ConfigSource configSource;
+
+    private LeadWithListInputPlugin leadWithListInputPlugin;
+
+    private MarketoRestClient mockMarketoRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        leadWithListInputPlugin = spy(new LeadWithListInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
+        mockMarketoRestClient = mock(MarketoRestClient.class);
+        doReturn(mockMarketoRestClient).when(leadWithListInputPlugin).createMarketoRestClient(any(LeadWithListInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testRun() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = mock(RecordPagingIterable.class);
+        RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = mock(RecordPagingIterable.class);
+        RecordPagingIterable<ObjectNode> mocklistRecords = mock(RecordPagingIterable.class);
+
+        when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
+        JavaType objectNodeListType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<ObjectNode> leads = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_response_full.json"), objectNodeListType);
+        List<ObjectNode> lists = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/list_reponse_full.json"), objectNodeListType);
+        when(mocklistRecords.iterator()).thenReturn(lists.iterator());
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
+        when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
+        when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
+        when(mockMarketoRestClient.getLists()).thenReturn(mocklistRecords);
+        List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
+        when(mockMarketoRestClient.getLeadsByList(anyString(), eq(fieldNameFromMarketoFields))).thenReturn(mockLeadEmptyRecordPagingIterable);
+        when(mockMarketoRestClient.getLeadsByList("1009", fieldNameFromMarketoFields)).thenReturn(mockLeadRecordPagingIterable);
+
+        LeadWithListInputPlugin.PluginTask task = configSource.loadConfig(LeadWithListInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = leadWithListInputPlugin.buildServiceResponseMapper(task);
+
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = mock(PageBuilder.class);
+        leadWithListInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        verify(mockMarketoRestClient, times(1)).getLists();
+        verify(mockMarketoRestClient, times(24)).getLeadsByList(anyString(), eq(fieldNameFromMarketoFields));
+        verify(mockMarketoRestClient, times(1)).describeLead();
+
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+
+        verify(mockPageBuilder, times(300)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
+        verify(mockPageBuilder, times(300)).setString(eq(embulkSchema.lookupColumn("mk_listId")), eq("1009"));
+
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        long actualValue = allValues.get(0);
+        assertEquals(103280L, actualValue);
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java

@@ -0,0 +1,101 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.*;
+
+/**
+ * Created by tai.khuu on 10/10/17.
+ */
+public class LeadWithProgramInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private ConfigSource configSource;
+
+    private LeadWithProgramInputPlugin leadWithProgramInputPlugin;
+
+    private MarketoRestClient mockMarketoRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        leadWithProgramInputPlugin = spy(new LeadWithProgramInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
+        mockMarketoRestClient = mock(MarketoRestClient.class);
+        doReturn(mockMarketoRestClient).when(leadWithProgramInputPlugin).createMarketoRestClient(any(LeadWithProgramInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testRun() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = mock(RecordPagingIterable.class);
+        RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = mock(RecordPagingIterable.class);
+        RecordPagingIterable<ObjectNode> mockProgramRecords = mock(RecordPagingIterable.class);
+
+        when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
+        JavaType objectNodeListType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<ObjectNode> leads = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_with_program_full.json"), objectNodeListType);
+        List<ObjectNode> programs = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/all_program_full.json"), objectNodeListType);
+        when(mockProgramRecords.iterator()).thenReturn(programs.iterator());
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
+        when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
+        when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
+        when(mockMarketoRestClient.getPrograms()).thenReturn(mockProgramRecords);
+        List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
+        when(mockMarketoRestClient.getLeadsByProgram(anyString(), eq(fieldNameFromMarketoFields))).thenReturn(mockLeadEmptyRecordPagingIterable);
+        when(mockMarketoRestClient.getLeadsByProgram("1003", fieldNameFromMarketoFields)).thenReturn(mockLeadRecordPagingIterable);
+
+        LeadWithProgramInputPlugin.PluginTask task = configSource.loadConfig(LeadWithProgramInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = leadWithProgramInputPlugin.buildServiceResponseMapper(task);
+
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = mock(PageBuilder.class);
+
+        leadWithProgramInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        verify(mockMarketoRestClient, times(1)).getPrograms();
+        verify(mockMarketoRestClient, times(3)).getLeadsByProgram(anyString(), eq(fieldNameFromMarketoFields));
+        verify(mockMarketoRestClient, times(1)).describeLead();
+
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+
+        verify(mockPageBuilder, times(1)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
+        verify(mockPageBuilder, times(1)).setString(eq(embulkSchema.lookupColumn("mk_programId")), eq("1003"));
+
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        long actualValue = allValues.get(0);
+        assertEquals(102519L, actualValue);
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java

@@ -0,0 +1,114 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Sets;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoInputPluginDelegate;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.spi.Schema;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.*;
+
+/**
+ * Created by khuutantaitai on 10/3/17.
+ */
+public class MarketoBaseBulkExtractInputPluginTest
+{
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private MarketoBaseBulkExtractInputPlugin<MarketoBaseBulkExtractInputPlugin.PluginTask> baseBulkExtractInputPlugin;
+
+    @Before
+    public void prepare()
+    {
+        baseBulkExtractInputPlugin = mock(MarketoBaseBulkExtractInputPlugin.class, Mockito.CALLS_REAL_METHODS);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void validateInputTaskError()
+    {
+        MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
+        when(pluginTask.getFromDate()).thenReturn(null);
+        baseBulkExtractInputPlugin.validateInputTask(pluginTask);
+    }
+
+    @Test()
+    public void validateInputTaskToDateLessThanJobStartTime()
+    {
+        Date fromDate = new Date(1504224000000L);
+        MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
+        when(pluginTask.getFromDate()).thenReturn(fromDate);
+        when(pluginTask.getFetchDays()).thenReturn(7);
+        baseBulkExtractInputPlugin.validateInputTask(pluginTask);
+        ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
+        verify(pluginTask, times(1)).setToDate(argumentCaptor.capture());
+        assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
+    }
+
+    @Test()
+    public void validateInputTaskToDateMoreThanJobStartTime()
+    {
+        Date fromDate = new Date(1504224000000L);
+        DateTime jobStartTime = new DateTime(1504396800000L);
+        MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
+        when(pluginTask.getFromDate()).thenReturn(fromDate);
+        when(pluginTask.getFetchDays()).thenReturn(7);
+        when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
+        baseBulkExtractInputPlugin.validateInputTask(pluginTask);
+        ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
+        verify(pluginTask, times(1)).setToDate(toDateArgumentCaptor.capture());
+        assertEquals(jobStartTime.minusHours(1).getMillis(), toDateArgumentCaptor.getValue().get().getTime());
+    }
+
+    @Test
+    public void getToDate() throws Exception
+    {
+        DateTime date = new DateTime(1505033728000L);
+        DateTime jobStartTime = new DateTime(1507625728000L);
+        MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoInputPluginDelegate.PluginTask.class);
+        when(pluginTask.getFromDate()).thenReturn(date.toDate());
+        when(pluginTask.getFetchDays()).thenReturn(30);
+        when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
+        DateTime toDate = baseBulkExtractInputPlugin.getToDate(pluginTask);
+        assertEquals(toDate, jobStartTime);
+    }
+
+    @Test
+    public void buildConfigDiff() throws Exception
+    {
+        TaskReport taskReport1 = mock(TaskReport.class);
+        TaskReport taskReport2 = mock(TaskReport.class);
+        when(taskReport1.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id1", "id2"));
+        when(taskReport2.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id3", "id4"));
+        when(taskReport1.get(Long.class, "latest_fetch_time")).thenReturn(1507539328000L);
+        when(taskReport2.get(Long.class, "latest_fetch_time")).thenReturn(1507625728000L);
+        MarketoInputPluginDelegate.PluginTask task = mock(MarketoInputPluginDelegate.PluginTask.class);
+        when(task.getIncremental()).thenReturn(true);
+        when(task.getIncrementalColumn()).thenReturn(Optional.of("createdAt"));
+        Date toDate = new Date(1507625728000L);
+        when(task.getToDate()).thenReturn(Optional.of(toDate));
+        ConfigDiff configDiff = baseBulkExtractInputPlugin.buildConfigDiff(task, mock(Schema.class), 1, Arrays.asList(taskReport1, taskReport2));
+        long latestFetchTime = configDiff.get(Long.class, "latest_fetch_time");
+        assertEquals(1507625728000L, latestFetchTime);
+        assertEquals(Sets.newHashSet("id3", "id4"), configDiff.get(Set.class, "latest_uids"));
+        DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+        assertEquals(df.format(toDate), configDiff.get(String.class, "from_date"));
+    }
+}

data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java

@@ -36,14 +36,12 @@ import static org.mockito.Mockito.*;
  */
 public class MarketoBaseRestClientTest
 {
-
     private static final String IDENTITY_END_POINT = "identityEndPoint";
 
     private static final int MARKETO_LIMIT_INTERVAL_MILIS = 1000;
 
     private MarketoBaseRestClient marketoBaseRestClient;
 
-
     private Jetty92RetryHelper mockJetty92;
 
     @Rule
@@ -55,12 +53,6 @@ public class MarketoBaseRestClientTest
         marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, mockJetty92);
     }
 
-    @Test
-    public void doGet() throws Exception
-    {
-
-    }
-
     @Test
     public void testGetAccessToken()
     {
@@ -77,7 +69,6 @@ public class MarketoBaseRestClientTest
     @Test
     public void testGetAccessTokenRequester()
     {
-
         ArgumentCaptor<Jetty92SingleRequester> jetty92SingleRequesterArgumentCaptor = ArgumentCaptor.forClass(Jetty92SingleRequester.class);
         when(mockJetty92.requestWithRetry(any(StringJetty92ResponseEntityReader.class), jetty92SingleRequesterArgumentCaptor.capture())).thenReturn("{\"access_token\": \"access_token\"}");
         String accessToken = marketoBaseRestClient.getAccessToken();
@@ -105,7 +96,8 @@ public class MarketoBaseRestClientTest
                 "}");
         try {
             marketoBaseRestClient.getAccessToken();
-        } catch (DataException ex) {
+        }
+        catch (DataException ex) {
             assertEquals("Bad client credentials", ex.getMessage());
             return;
         }
@@ -163,7 +155,6 @@ public class MarketoBaseRestClientTest
         verify(mockRequest, times(1)).header(eq("Authorization"), eq("Bearer access_token"));
         verify(mockRequest, times(1)).param(eq("param"), eq("param1"));
         verify(mockRequest, times(1)).content(eq(contentProvider), eq("application/json"));
-
     }
 
     @Test
@@ -188,7 +179,6 @@ public class MarketoBaseRestClientTest
         jetty92SingleRequester.requestOnce(client, listener);
         assertTrue(jetty92SingleRequester.toRetry(createHttpResponseException(502)));
 
-
         assertFalse(jetty92SingleRequester.toRetry(createHttpResponseException(400)));
 
         assertFalse(jetty92SingleRequester.toRetry(createMarketoAPIException("ERR", "ERR")));
@@ -197,7 +187,6 @@ public class MarketoBaseRestClientTest
         assertTrue(jetty92SingleRequester.toRetry(createMarketoAPIException("602", "")));
 
         verify(mockJetty92, times(2)).requestWithRetry(any(StringJetty92ResponseEntityReader.class), any(Jetty92SingleRequester.class));
-
     }
 
     private HttpResponseException createHttpResponseException(int statusCode)
@@ -216,5 +205,4 @@ public class MarketoBaseRestClientTest
         marketoError.setMessage(error);
         return new MarketoAPIException(Lists.newArrayList(marketoError));
     }
-
-}
+}