embulk-input-marketo-through-proxy 0.6.20

data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java

@@ -0,0 +1,73 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Created by tai.khuu on 10/10/17.
+ */
+public class CampaignInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private ConfigSource configSource;
+
+    private CampaignInputPlugin campaignInputPlugin;
+
+    private MarketoRestClient mockMarketoRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        campaignInputPlugin = Mockito.spy(new CampaignInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
+        mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockMarketoRestClient).when(campaignInputPlugin).createMarketoRestClient(Mockito.any(CampaignInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testRun() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/campaign_response_full.json"), javaType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+        Mockito.when(mockMarketoRestClient.getCampaign()).thenReturn(mockRecordPagingIterable);
+        CampaignInputPlugin.PluginTask task = configSource.loadConfig(CampaignInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = campaignInputPlugin.buildServiceResponseMapper(task);
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        campaignInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCampaign();
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Mockito.verify(mockPageBuilder, Mockito.times(10)).setLong(Mockito.eq(embulkSchema.lookupColumn("id")), longArgumentCaptor.capture());
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        assertArrayEquals(new Long[]{1003L, 1004L, 1005L, 1006L, 1007L, 1008L, 1029L, 1048L, 1051L, 1065L}, allValues.toArray());
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java

@@ -0,0 +1,175 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+
+public class CustomObjectInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private CustomObjectInputPlugin customObjectInputPlugin;
+
+    private ConfigSource configSource;
+
+    private MarketoRestClient mockMarketoRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        customObjectInputPlugin = Mockito.spy(new CustomObjectInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/custom_object_config.yaml"));
+        mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockMarketoRestClient).when(customObjectInputPlugin).createMarketoRestClient(Mockito.any(CustomObjectInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testValidPluginTask()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource.loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        // should not cause exception
+        customObjectInputPlugin.validateInputTask(pluginTask);
+    }
+
+    @Test
+    public void testCustomObjectFilterTypeError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource
+                .set("custom_object_filter_type", "")
+                .loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        Assert.assertThrows(ConfigException.class, () -> customObjectInputPlugin.validateInputTask(pluginTask));
+    }
+
+    @Test
+    public void testCustomObjectAPINameError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource
+                .set("custom_object_api_name", "")
+                .loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        Assert.assertThrows(ConfigException.class, () -> customObjectInputPlugin.validateInputTask(pluginTask));
+    }
+
+    @Test
+    public void testFromValueGreaterThanToValueError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource
+                .set("custom_object_filter_from_value", "10")
+                .set("custom_object_filter_to_value", "5").loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        Assert.assertThrows(ConfigException.class, () -> customObjectInputPlugin.validateInputTask(pluginTask));
+    }
+
+    @Test
+    public void testEmptyStringFilterValues()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource
+                .set("custom_object_filter_values", "").loadConfig(CustomObjectInputPlugin.PluginTask.class);
+
+        Assert.assertThrows(ConfigException.class, () -> customObjectInputPlugin.validateInputTask(pluginTask));
+    }
+
+    @Test
+    public void testAllEmptyStringFilterValues()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = configSource
+                .set("custom_object_filter_values", ",, , ").loadConfig(CustomObjectInputPlugin.PluginTask.class);
+
+        Assert.assertThrows(ConfigException.class, () -> customObjectInputPlugin.validateInputTask(pluginTask));
+    }
+
+    @Test
+    public void testRunStringFilterValues() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = mockAndGetResponse();
+        Mockito.when(mockMarketoRestClient.getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(mockRecordPagingIterable);
+
+        CustomObjectInputPlugin.PluginTask task = configSource.set("custom_object_filter_values", "value1,value2,value3,value4").loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = customObjectInputPlugin.buildServiceResponseMapper(task);
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        customObjectInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+
+        verifyAfterRun(mapper, mockPageBuilder);
+    }
+
+    @Test
+    public void testRunWithFilterRange() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = mockAndGetResponse();
+        Mockito.when(mockMarketoRestClient.getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(mockRecordPagingIterable);
+
+        CustomObjectInputPlugin.PluginTask task = configSource.loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = customObjectInputPlugin.buildServiceResponseMapper(task);
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        customObjectInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString());
+
+        verifyAfterRun(mapper, mockPageBuilder);
+    }
+
+    @Test
+    public void testRunFromOnlyFilter() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = mockAndGetResponse();
+        Mockito.when(mockMarketoRestClient.getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.isNull())).thenReturn(mockRecordPagingIterable);
+
+        CustomObjectInputPlugin.PluginTask task = configSource.remove("custom_object_filter_to_value").loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = customObjectInputPlugin.buildServiceResponseMapper(task);
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        customObjectInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.isNull());
+
+        verifyAfterRun(mapper, mockPageBuilder);
+    }
+
+    private void verifyAfterRun(ServiceResponseMapper<? extends ValueLocator> mapper, PageBuilder mockPageBuilder)
+    {
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeCustomObject(Mockito.anyString());
+        Schema schema = mapper.getEmbulkSchema();
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Mockito.verify(mockPageBuilder, Mockito.times(3)).setLong(Mockito.eq(schema.lookupColumn("mk_id")), longArgumentCaptor.capture());
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        assertArrayEquals(new Long[]{1L, 2L, 3L}, allValues.toArray());
+    }
+
+    private RecordPagingIterable<ObjectNode> mockAndGetResponse() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/custom_object_response_full.json"), javaType);
+        JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/custom_object_describe_marketo_fields_full.json"), marketoFieldsType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+        Mockito.when(mockMarketoRestClient.describeCustomObject(Mockito.anyString())).thenReturn(marketoFields);
+        return mockRecordPagingIterable;
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java

@@ -0,0 +1,102 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.BulkExtractRangeHeader;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
+/**
+ * Created by khuutantaitai on 10/3/17.
+ */
+public class LeadBulkExtractInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private LeadBulkExtractInputPlugin bulkExtractInputPlugin;
+
+    private ConfigSource configSource;
+
+    private MarketoRestClient mockMarketoRestclient;
+
+    @Before
+    public void prepare() throws IOException
+    {
+        bulkExtractInputPlugin = Mockito.spy(new LeadBulkExtractInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/lead_bulk_extract_config.yaml"));
+        mockMarketoRestclient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockMarketoRestclient).when(bulkExtractInputPlugin).createMarketoRestClient(any(LeadBulkExtractInputPlugin.PluginTask.class));
+    }
+
+    @Test
+    public void testRun() throws InterruptedException, IOException
+    {
+        LeadBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(LeadBulkExtractInputPlugin.PluginTask.class);
+        OffsetDateTime startDate = OffsetDateTime.ofInstant(task.getFromDate().toInstant(), ZoneOffset.UTC);
+        PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
+        String exportId1 = "exportId1";
+        String exportId2 = "exportId2";
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), javaType);
+        List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
+        Mockito.when(mockMarketoRestclient.describeLead()).thenReturn(marketoFields);
+        Mockito.when(mockMarketoRestclient.createLeadBulkExtract(any(Date.class), any(Date.class), any(List.class), any(String.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
+        Mockito.when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/lead_extract1.csv"));
+        Mockito.when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/leads_extract2.csv"));
+        ServiceResponseMapper<? extends ValueLocator> mapper = bulkExtractInputPlugin.buildServiceResponseMapper(task);
+        bulkExtractInputPlugin.validateInputTask(task);
+        TaskReport taskReport = bulkExtractInputPlugin.ingestServiceData(task, mapper.createRecordImporter(), 1, pageBuilder);
+        ArgumentCaptor<Long> argumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Column idColumn = mapper.getEmbulkSchema().lookupColumn("mk_id");
+        Mockito.verify(pageBuilder, Mockito.times(19)).setLong(eq(idColumn), argumentCaptor.capture());
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startLeadBulkExtract(eq(exportId1));
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startLeadBulkExtract(eq(exportId2));
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
+        String filterField = "createdAt";
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(Date.from(startDate.toInstant()),
+                Date.from(startDate.plusDays(30).toInstant()), fieldNameFromMarketoFields, filterField);
+        OffsetDateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(Date.from(startDate2.toInstant()),
+                Date.from(startDate.plusDays(task.getFetchDays()).toInstant()), fieldNameFromMarketoFields, filterField);
+        List<Long> leadIds = argumentCaptor.getAllValues();
+        Assert.assertEquals(19, leadIds.size());
+        ConfigDiff configDiff = bulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
+        DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+        Assert.assertEquals(df.format(Date.from(startDate.plusDays(task.getFetchDays()).toInstant())), configDiff.get(String.class, "from_date"));
+        Assert.assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
+    }
+}

data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java

@@ -0,0 +1,119 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Schema;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 5/24/18.
+ */
+public class LeadServiceResponseMapperBuilderTest
+{
+    private LeadServiceResponseMapperBuilder<LeadServiceResponseMapperBuilder.PluginTask> leadServiceResponseMapperBuilder;
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private LeadServiceResponseMapperBuilder.PluginTask pluginTask;
+
+    private ConfigSource configSource;
+
+    private MarketoService marketoService;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        ConfigLoader configLoader = embulkTestRuntime.getExec().getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYamlString("--- \n" +
+                "account_id: 389-FLQ-873\n" +
+                "client_id: 87bdd058-12ff-49b8-8d29-c0518669e59a\n" +
+                "client_secret: 2uauyk43zzwjlUZwboZVxQ3SMTHe8ILY\n" +
+                "included_fields: \n" +
+                "  - company\n" +
+                "  - site\n" +
+                "  - billingstreet\n" +
+                "  - billingcity\n" +
+                "  - billingstate\n" +
+                "  - billingcountry\n" +
+                "  - billingpostalCode\n" +
+                "  - website\n" +
+                "  - mainphone\n" +
+                "  - annualrevenue\n" +
+                "  - numberofemployees\n" +
+                "  - industry\n" +
+                "  - siccode\n" +
+                "  - mktoCompanyNotes\n" +
+                "  - externalcompanyId\n" +
+                "  - id\n" +
+                "target: all_lead_with_list_id\n");
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        marketoService = Mockito.mock(MarketoService.class);
+        JavaType marketoFieldsType = MarketoUtils.OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = MarketoUtils.OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
+        Mockito.when(marketoService.describeLead()).thenReturn(marketoFields);
+    }
+
+    @Test
+    public void buildServiceResponseMapper() throws Exception
+    {
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        ServiceResponseMapper<? extends ValueLocator> serviceResponseMapper = leadServiceResponseMapperBuilder.buildServiceResponseMapper(pluginTask);
+        Assert.assertFalse(pluginTask.getExtractedFields().isEmpty());
+        Assert.assertEquals(16, pluginTask.getExtractedFields().size());
+        Schema embulkSchema = serviceResponseMapper.getEmbulkSchema();
+        Assert.assertEquals(16, embulkSchema.getColumns().size());
+        Assert.assertEquals("mk_billingStreet", embulkSchema.getColumns().get(2).getName());
+        Assert.assertEquals("mk_company", embulkSchema.getColumn(0).getName());
+        Assert.assertEquals("mk_billingStreet", embulkSchema.getColumn(2).getName());
+        Assert.assertEquals("mk_id", embulkSchema.getColumn(15).getName());
+    }
+
+    @Test
+    public void getLeadColumnsIncludedEmpty() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertEquals(129, leadColumns.size());
+    }
+
+    @Test
+    public void getLeadColumnsIncluded1() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[\"company\",\"incorrect_value\"]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertEquals(1, leadColumns.size());
+        Assert.assertEquals("company", leadColumns.get(0).getName());
+    }
+
+    @Test
+    public void getLeadColumnsIncluded2() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[\"company\",\"incorrect_value\"]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        marketoService = Mockito.mock(MarketoService.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        Mockito.when(marketoService.describeLead()).thenReturn(new ArrayList<MarketoField>());
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertTrue(leadColumns.isEmpty());
+    }
+}