embulk-input-marketo 0.6.0 → 0.6.1.alpha.1

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -64,6 +64,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
 
     private static final int MARKETO_MAX_RANGE_EXTRACT = 30;
 
+    private static final String IMPORTED = "imported";
+
     public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
     {
         @Config("from_date")
@@ -145,6 +147,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         Long currentLatestFetchTime = 0L;
         Set latestUIds = null;
         String incrementalColumn = task.getIncrementalColumn().orNull();
+        int imported = 0;
         if (incrementalColumn != null && task.getIncremental()) {
             DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
             for (TaskReport taskReport : taskReports) {
@@ -156,12 +159,21 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     currentLatestFetchTime = latestFetchTime;
                     latestUIds = taskReport.get(Set.class, LATEST_UID_LIST);
                 }
-                else if (currentLatestFetchTime == latestFetchTime) {
+                else if (currentLatestFetchTime.equals(latestFetchTime)) {
                     latestUIds.addAll(taskReport.get(Set.class, LATEST_UID_LIST));
                 }
+                if (taskReport.has(IMPORTED)) {
+                    imported = imported + taskReport.get(Integer.class, IMPORTED);
+                }
             }
             // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
-            configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
+            if (imported == 0) {
+                configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
+            }
+            else {
+            // Otherwise it's should start from the currentLastFetchTime. If Marketo could guaranteed that they always return data of full range we could just set it to_date
+                configDiff.set(FROM_DATE, df.format(new Date(currentLatestFetchTime)));
+            }
             configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
             configDiff.set(LATEST_UID_LIST, latestUIds);
         }
@@ -193,6 +205,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 if (Exec.isPreview()) {
                     csvRecords = Iterators.limit(csvRecords, PREVIEW_RECORD_LIMIT);
                 }
+                int imported = 0;
                 while (csvRecords.hasNext()) {
                     Map<String, String> csvRecord = csvRecords.next();
                     if (task.getIncremental()) {
@@ -221,12 +234,13 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                             }
                         }
                     }
-
                     ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
                     recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
+                    imported = imported + 1;
                 }
                 taskReport.set(LATEST_FETCH_TIME, currentTimestamp);
                 taskReport.set(LATEST_UID_LIST, latestUids);
+                taskReport.set(IMPORTED, imported);
                 return taskReport;
             }
         }

data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java

@@ -0,0 +1,26 @@
+package org.embulk.input.marketo.model;
+
+/**
+ * Created by tai.khuu on 10/12/17.
+ */
+public class BulkExtractRangeHeader
+{
+    private Long start;
+    private Long end;
+
+    public BulkExtractRangeHeader(long start)
+    {
+        this.start = start;
+    }
+
+    public BulkExtractRangeHeader(long start, long end)
+    {
+        this.start = start;
+        this.end = end;
+    }
+
+    public String toRangeHeaderValue()
+    {
+        return "bytes=" + start + "-" + (end != null ? end : "");
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -141,7 +141,12 @@ public class MarketoBaseRestClient implements AutoCloseable
         if (content != null) {
             contentProvider = new StringContentProvider(APPLICATION_JSON, content, StandardCharsets.UTF_8);
         }
-        return doRequest(target, HttpMethod.POST, headers, params, contentProvider, responseReader);
+        return doPost(target, headers, params, responseReader, contentProvider);
+    }
+
+    protected <T> T doPost(final String target, final Map<String, String> headers, final Multimap<String, String> params, Jetty92ResponseReader<T> responseReader, final ContentProvider content)
+    {
+        return doRequest(target, HttpMethod.POST, headers, params, content, responseReader);
     }
 
     protected <T> T doRequest(final String target, final HttpMethod method, final Map<String, String> headers, final Multimap<String, String> params, final ContentProvider contentProvider, Jetty92ResponseReader<T> responseReader)
@@ -152,7 +157,6 @@ public class MarketoBaseRestClient implements AutoCloseable
             public void requestOnce(HttpClient client, Response.Listener responseListener)
             {
                 Request request = client.newRequest(target).method(method);
-
                 if (headers != null) {
                     for (String key : headers.keySet()) {
                         request.header(key, headers.get(key));
@@ -187,7 +191,8 @@ public class MarketoBaseRestClient implements AutoCloseable
                 if (exception instanceof ExecutionException) {
                     this.toRetry((Exception) exception.getCause());
                 }
-                if (exception instanceof EOFException) {
+                //Anything that is EOFException or cause by EOF exception
+                if (exception instanceof EOFException || exception.getCause() instanceof EOFException) {
                     return true;
                 }
                 if (exception instanceof MarketoAPIException) {

data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java

@@ -29,13 +29,13 @@ public class MarketoInputStreamResponseEntityReader implements Jetty92ResponseRe
 
     public MarketoInputStreamResponseEntityReader(long timeout)
     {
-        this.listener = new InputStreamResponseListener();
         this.timeout = timeout;
     }
 
     @Override
     public Response.Listener getListener()
     {
+        this.listener = new InputStreamResponseListener();
         return this.listener;
     }
 

data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java

@@ -36,7 +36,6 @@ public class MarketoResponseJetty92EntityReader<T> implements Jetty92ResponseRea
 
     public MarketoResponseJetty92EntityReader(long timeout)
     {
-        this.listener = new InputStreamResponseListener();
         this.timeout = timeout;
         javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(MarketoResponse.class, MarketoResponse.class, ObjectNode.class);
     }
@@ -51,7 +50,8 @@ public class MarketoResponseJetty92EntityReader<T> implements Jetty92ResponseRea
     @Override
     public Response.Listener getListener()
     {
-        return listener;
+        this.listener = new InputStreamResponseListener();
+        return this.listener;
     }
 
     @Override
@@ -83,8 +83,6 @@ public class MarketoResponseJetty92EntityReader<T> implements Jetty92ResponseRea
         InputStream inputStream = this.listener.getInputStream();
         try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) {
             String reponseContent = CharStreams.toString(inputStreamReader);
-            //Reset this listener
-            this.listener = new InputStreamResponseListener();
             return reponseContent;
         }
     }

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -6,11 +6,13 @@ import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Multimap;
-import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jetty.client.util.FormContentProvider;
+import org.eclipse.jetty.util.Fields;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.BulkExtractRangeHeader;
 import org.embulk.input.marketo.model.MarketoBulkExtractRequest;
 import org.embulk.input.marketo.model.MarketoError;
 import org.embulk.input.marketo.model.MarketoField;
@@ -45,6 +47,12 @@ public class MarketoRestClient extends MarketoBaseRestClient
 
     private static final String MAX_RETURN = "maxReturn";
 
+    private static final String MAX_BATCH_SIZE = "300";
+
+    private static final String DEFAULT_MAX_RETURN = "200";
+
+    private static final String RANGE_HEADER = "Range";
+
     private String endPoint;
 
     private Integer batchSize;
@@ -119,9 +127,11 @@ public class MarketoRestClient extends MarketoBaseRestClient
         for (int i = 0; i < fields.size(); i++) {
             ObjectNode field = fields.get(i);
             String dataType = field.get("dataType").asText();
-            ObjectNode restField = (ObjectNode) field.get("rest");
-            String name = restField.get("name").asText();
-            marketoFields.add(new MarketoField(name, dataType));
+            if (field.has("rest")) {
+                ObjectNode restField = (ObjectNode) field.get("rest");
+                String name = restField.get("name").asText();
+                marketoFields.add(new MarketoField(name, dataType));
+            }
         }
         return marketoFields;
     }
@@ -261,42 +271,48 @@ public class MarketoRestClient extends MarketoBaseRestClient
         }
     }
 
-    public InputStream getLeadBulkExtractResult(String exportId)
+    public InputStream getLeadBulkExtractResult(String exportId, BulkExtractRangeHeader bulkExtractRangeHeader)
     {
-        return getBulkExtractResult(MarketoRESTEndpoint.GET_LEAD_EXPORT_RESULT, exportId);
+        return getBulkExtractResult(MarketoRESTEndpoint.GET_LEAD_EXPORT_RESULT, exportId, bulkExtractRangeHeader);
     }
 
-    public InputStream getActivitiesBulkExtractResult(String exportId)
+    public InputStream getActivitiesBulkExtractResult(String exportId, BulkExtractRangeHeader bulkExtractRangeHeader)
     {
-        return getBulkExtractResult(MarketoRESTEndpoint.GET_ACTIVITY_EXPORT_RESULT, exportId);
+        return getBulkExtractResult(MarketoRESTEndpoint.GET_ACTIVITY_EXPORT_RESULT, exportId, bulkExtractRangeHeader);
     }
 
-    private InputStream getBulkExtractResult(MarketoRESTEndpoint endpoint, String exportId)
+    private InputStream getBulkExtractResult(MarketoRESTEndpoint endpoint, String exportId, BulkExtractRangeHeader bulkExtractRangeHeader)
     {
-        return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), null, null, new MarketoInputStreamResponseEntityReader(READ_TIMEOUT_MILLIS));
+        LOGGER.info("Download bulk export job [{}]", exportId);
+        Map<String, String> headers = new HashMap<>();
+        if (bulkExtractRangeHeader != null) {
+            headers.put(RANGE_HEADER, bulkExtractRangeHeader.toRangeHeaderValue());
+            LOGGER.info("Range header value [{}]", bulkExtractRangeHeader.toRangeHeaderValue());
+        }
+        return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), headers, null, new MarketoInputStreamResponseEntityReader(READ_TIMEOUT_MILLIS));
     }
 
     public RecordPagingIterable<ObjectNode> getLists()
     {
-        return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_LISTS.getEndpoint(), null, ObjectNode.class);
+        return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_LISTS.getEndpoint(), new ImmutableListMultimap.Builder<String, String>().put(BATCH_SIZE, MAX_BATCH_SIZE).build(), ObjectNode.class);
     }
 
     public RecordPagingIterable<ObjectNode> getPrograms()
     {
-        return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), null, ObjectNode.class);
+        return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), new ImmutableListMultimap.Builder<String, String>().put(MAX_RETURN, DEFAULT_MAX_RETURN).build(), ObjectNode.class);
     }
 
-    public RecordPagingIterable<ObjectNode> getLeadsByProgram(String programId, List<String> fieldNames)
+    public RecordPagingIterable<ObjectNode> getLeadsByProgram(String programId, String fieldNames)
     {
         Multimap<String, String> multimap = ArrayListMultimap.create();
-        multimap.put("fields", StringUtils.join(fieldNames, ","));
+        multimap.put("fields", fieldNames);
         return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_LEADS_BY_PROGRAM.getEndpoint(new ImmutableMap.Builder().put("program_id", programId).build()), multimap, ObjectNode.class);
     }
 
-    public RecordPagingIterable<ObjectNode> getLeadsByList(String listId, List<String> fieldNames)
+    public RecordPagingIterable<ObjectNode> getLeadsByList(String listId, String fieldNames)
     {
         Multimap<String, String> multimap = ArrayListMultimap.create();
-        multimap.put("fields", StringUtils.join(fieldNames, ","));
+        multimap.put("fields", fieldNames);
         return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_LEADS_BY_LIST.getEndpoint(new ImmutableMap.Builder().put("list_id", listId).build()), multimap, ObjectNode.class);
     }
 
@@ -348,17 +364,28 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 return getTokenPage(null);
             }
 
+            @SuppressWarnings("unchecked")
             private RecordPagingIterable.TokenPage<T> getTokenPage(RecordPagingIterable.TokenPage page)
             {
                 ImmutableListMultimap.Builder params = new ImmutableListMultimap.Builder<>();
+                params.put("_method", "GET");
+                Fields fields = new Fields();
                 if (page != null) {
-                    params.put(NEXT_PAGE_TOKEN, page.getNextPageToken());
+                    fields.add(NEXT_PAGE_TOKEN, page.getNextPageToken());
                 }
-                params.put(BATCH_SIZE, String.valueOf(batchSize));
+                fields.add(BATCH_SIZE, String.valueOf(batchSize));
                 if (parameters != null) {
-                    params.putAll(parameters);
+                    for (String key : parameters.keySet()) {
+                        //params that is passed in should overwrite default
+                        fields.remove(key);
+                        for (String value : parameters.get(key)) {
+                            fields.add(key, value);
+                        }
+                    }
                 }
-                MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(READ_TIMEOUT_MILLIS, recordClass));
+                //Let do GET Disguise in POST here to overcome Marketo URI Too long error
+                FormContentProvider formContentProvider = new FormContentProvider(fields);
+                MarketoResponse<T> marketoResponse = doPost(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(READ_TIMEOUT_MILLIS, recordClass), formContentProvider);
                 return new RecordPagingIterable.TokenPage<>(marketoResponse.getResult(), marketoResponse.getNextPageToken(), marketoResponse.getNextPageToken() != null);
             }
         });

data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java

@@ -4,20 +4,24 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.io.ByteStreams;
 import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.marketo.model.BulkExtractRangeHeader;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.util.*;
-
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
 
 /**
  * Created by tai.khuu on 10/9/17.
@@ -33,7 +37,7 @@ public class MarketoServiceImplTest
     @Before
     public void prepare()
     {
-        mockMarketoRestClient = mock(MarketoRestClient.class);
+        mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
         marketoService = new MarketoServiceImpl(mockMarketoRestClient);
     }
 
@@ -45,13 +49,13 @@ public class MarketoServiceImplTest
         List<String> extractedFields = Arrays.asList("field1", "field2");
         String filerField = "field1";
         String exportId = "exportId";
-        when(mockMarketoRestClient.createLeadBulkExtract(eq(startDate), eq(endDate), eq(extractedFields), eq(filerField))).thenReturn(exportId);
+        Mockito.when(mockMarketoRestClient.createLeadBulkExtract(Mockito.eq(startDate), Mockito.eq(endDate), Mockito.eq(extractedFields), Mockito.eq(filerField))).thenReturn(exportId);
         ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
-        when(mockMarketoRestClient.getLeadBulkExtractResult(eq(exportId))).thenReturn(byteArrayInputStream);
+        Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
         File file = marketoService.extractLead(startDate, endDate, extractedFields, filerField, 1, 3);
-        assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
-        verify(mockMarketoRestClient, times(1)).startLeadBulkExtract(eq(exportId));
-        verify(mockMarketoRestClient, times(1)).waitLeadExportJobComplete(eq(exportId), eq(1), eq(3));
+        Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startLeadBulkExtract(Mockito.eq(exportId));
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).waitLeadExportJobComplete(Mockito.eq(exportId), Mockito.eq(1), Mockito.eq(3));
     }
 
     @Test
@@ -60,100 +64,99 @@ public class MarketoServiceImplTest
         Date startDate = new Date(1507223374000L);
         Date endDate = new Date(1507655374000L);
         String exportId = "exportId";
-        when(mockMarketoRestClient.createActivityExtract(eq(startDate), eq(endDate))).thenReturn(exportId);
+        Mockito.when(mockMarketoRestClient.createActivityExtract(Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
         ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
-        when(mockMarketoRestClient.getActivitiesBulkExtractResult(eq(exportId))).thenReturn(byteArrayInputStream);
+        Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
         File file = marketoService.extractAllActivity(startDate, endDate, 1, 3);
-        assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
-        verify(mockMarketoRestClient, times(1)).startActitvityBulkExtract(eq(exportId));
-        verify(mockMarketoRestClient, times(1)).waitActitvityExportJobComplete(eq(exportId), eq(1), eq(3));
+        Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId));
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId), Mockito.eq(1), Mockito.eq(3));
     }
 
     @Test
     public void getAllListLead() throws Exception
     {
         List<String> extractFields = Arrays.asList("field1", "field2");
-        RecordPagingIterable<ObjectNode> listObjectNodes = mock(RecordPagingIterable.class);
-        Iterator listIterator = mock(Iterator.class);
-        when(listIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
-        when(listIterator.next()).thenReturn(OBJECT_MAPPER.readTree("{\"id\":1}")).thenReturn(OBJECT_MAPPER.readTree("{\"id\":2}"));
-        when(listObjectNodes.iterator()).thenReturn(listIterator);
+        RecordPagingIterable<ObjectNode> listObjectNodes = Mockito.mock(RecordPagingIterable.class);
+        Iterator listIterator = Mockito.mock(Iterator.class);
+        Mockito.when(listIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
+        Mockito.when(listIterator.next()).thenReturn(OBJECT_MAPPER.readTree("{\"id\":1}")).thenReturn(OBJECT_MAPPER.readTree("{\"id\":2}"));
+        Mockito.when(listObjectNodes.iterator()).thenReturn(listIterator);
         List<ObjectNode> leadList1 = new ArrayList<>();
         leadList1.add((ObjectNode) OBJECT_MAPPER.readTree("{\"id\":\"lead1\"}"));
         List<ObjectNode> leadList2 = new ArrayList<>();
         leadList2.add((ObjectNode) OBJECT_MAPPER.readTree("{\"id\":\"lead2\"}"));
-        when(mockMarketoRestClient.getLists()).thenReturn(listObjectNodes);
-        RecordPagingIterable leadIterable1 = mock(RecordPagingIterable.class);
-        RecordPagingIterable leadsIterable2 = mock(RecordPagingIterable.class);
-        when(leadIterable1.iterator()).thenReturn(leadList1.iterator());
-        when(leadsIterable2.iterator()).thenReturn(leadList2.iterator());
-        when(mockMarketoRestClient.getLeadsByList(eq("1"), eq(extractFields))).thenReturn(leadIterable1);
-        when(mockMarketoRestClient.getLeadsByList(eq("2"), eq(extractFields))).thenReturn(leadsIterable2);
+        Mockito.when(mockMarketoRestClient.getLists()).thenReturn(listObjectNodes);
+        RecordPagingIterable leadIterable1 = Mockito.mock(RecordPagingIterable.class);
+        RecordPagingIterable leadsIterable2 = Mockito.mock(RecordPagingIterable.class);
+        Mockito.when(leadIterable1.iterator()).thenReturn(leadList1.iterator());
+        Mockito.when(leadsIterable2.iterator()).thenReturn(leadList2.iterator());
+        Mockito.when(mockMarketoRestClient.getLeadsByList(Mockito.eq("1"), Mockito.eq("field1,field2"))).thenReturn(leadIterable1);
+        Mockito.when(mockMarketoRestClient.getLeadsByList(Mockito.eq("2"), Mockito.eq("field1,field2"))).thenReturn(leadsIterable2);
         Iterable<ObjectNode> allListLead = marketoService.getAllListLead(extractFields);
-        assertEquals(leadList1.get(0), allListLead.iterator().next());
-        assertEquals(leadList2.get(0), allListLead.iterator().next());
+        Assert.assertEquals(leadList1.get(0), allListLead.iterator().next());
+        Assert.assertEquals(leadList2.get(0), allListLead.iterator().next());
     }
 
     @Test
     public void getAllProgramLead() throws Exception
     {
-        List<String> extractFields = Arrays.asList("field1", "field2");
-        RecordPagingIterable<ObjectNode> listObjectNodes = mock(RecordPagingIterable.class);
-        Iterator listIterator = mock(Iterator.class);
-        when(listIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
-        when(listIterator.next()).thenReturn(OBJECT_MAPPER.readTree("{\"id\":1}")).thenReturn(OBJECT_MAPPER.readTree("{\"id\":2}"));
-        when(listObjectNodes.iterator()).thenReturn(listIterator);
+        RecordPagingIterable<ObjectNode> listObjectNodes = Mockito.mock(RecordPagingIterable.class);
+        Iterator listIterator = Mockito.mock(Iterator.class);
+        Mockito.when(listIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
+        Mockito.when(listIterator.next()).thenReturn(OBJECT_MAPPER.readTree("{\"id\":1}")).thenReturn(OBJECT_MAPPER.readTree("{\"id\":2}"));
+        Mockito.when(listObjectNodes.iterator()).thenReturn(listIterator);
         List<ObjectNode> leadList1 = new ArrayList<>();
         leadList1.add((ObjectNode) OBJECT_MAPPER.readTree("{\"id\":\"lead1\"}"));
         List<ObjectNode> leadList2 = new ArrayList<>();
         leadList2.add((ObjectNode) OBJECT_MAPPER.readTree("{\"id\":\"lead2\"}"));
-        when(mockMarketoRestClient.getPrograms()).thenReturn(listObjectNodes);
-        RecordPagingIterable leadIterable1 = mock(RecordPagingIterable.class);
-        RecordPagingIterable leadsIterable2 = mock(RecordPagingIterable.class);
-        when(leadIterable1.iterator()).thenReturn(leadList1.iterator());
-        when(leadsIterable2.iterator()).thenReturn(leadList2.iterator());
-        when(mockMarketoRestClient.getLeadsByProgram(eq("1"), eq(extractFields))).thenReturn(leadIterable1);
-        when(mockMarketoRestClient.getLeadsByProgram(eq("2"), eq(extractFields))).thenReturn(leadsIterable2);
-        Iterable<ObjectNode> allListLead = marketoService.getAllProgramLead(extractFields);
-        assertEquals(leadList1.get(0), allListLead.iterator().next());
-        assertEquals(leadList2.get(0), allListLead.iterator().next());
+        Mockito.when(mockMarketoRestClient.getPrograms()).thenReturn(listObjectNodes);
+        RecordPagingIterable leadIterable1 = Mockito.mock(RecordPagingIterable.class);
+        RecordPagingIterable leadsIterable2 = Mockito.mock(RecordPagingIterable.class);
+        Mockito.when(leadIterable1.iterator()).thenReturn(leadList1.iterator());
+        Mockito.when(leadsIterable2.iterator()).thenReturn(leadList2.iterator());
+        Mockito.when(mockMarketoRestClient.getLeadsByProgram(Mockito.eq("1"), Mockito.eq("field1,field2"))).thenReturn(leadIterable1);
+        Mockito.when(mockMarketoRestClient.getLeadsByProgram(Mockito.eq("2"), Mockito.eq("field1,field2"))).thenReturn(leadsIterable2);
+        Iterable<ObjectNode> allListLead = marketoService.getAllProgramLead(Arrays.asList("field1", "field2"));
+        Assert.assertEquals(leadList1.get(0), allListLead.iterator().next());
+        Assert.assertEquals(leadList2.get(0), allListLead.iterator().next());
     }
 
     @Test
     public void getCampaign() throws Exception
     {
         marketoService.getCampaign();
-        verify(mockMarketoRestClient, times(1)).getCampaign();
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCampaign();
     }
 
     @Test
     public void describeLead() throws Exception
     {
         marketoService.describeLead();
-        verify(mockMarketoRestClient, times(1)).describeLead();
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
     }
 
     @Test
     public void describeLeadByProgram() throws Exception
     {
         List<MarketoField> marketoFields = new ArrayList<>();
-        when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
+        Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
         marketoService.describeLeadByProgram();
-        verify(mockMarketoRestClient, times(1)).describeLead();
-        assertEquals(1, marketoFields.size());
-        assertEquals("programId", marketoFields.get(0).getName());
-        assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
+        Assert.assertEquals(1, marketoFields.size());
+        Assert.assertEquals("programId", marketoFields.get(0).getName());
+        Assert.assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
     }
 
     @Test
     public void describeLeadByLists() throws Exception
     {
         List<MarketoField> marketoFields = new ArrayList<>();
-        when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
+        Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
         marketoService.describeLeadByLists();
-        verify(mockMarketoRestClient, times(1)).describeLead();
-        assertEquals(1, marketoFields.size());
-        assertEquals("listId", marketoFields.get(0).getName());
-        assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
+        Assert.assertEquals(1, marketoFields.size());
+        Assert.assertEquals("listId", marketoFields.get(0).getName());
+        Assert.assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
     }
 }