embulk-input-marketo 0.6.9 → 0.6.10.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0cae436fbdd33d197c7acb3e72ec4d5e2f5b00e
-  data.tar.gz: 9077aadd1cf80fecfeef1ba687f1827a01d64416
+  metadata.gz: 22645790a18bbe5578b7146e488167fe0139a1a1
+  data.tar.gz: b26ea8361cbe4f5d32796eb7295250becc11497d
 SHA512:
-  metadata.gz: 511670f368834d3d7d0a47a13d0a99391f3f7a0935e02d2011ef4d18e0d15ab6faa487358056ae2bcddf4fcdb1955e84b21639bae5f4eda5834b9d9f414b50a8
-  data.tar.gz: 8ae2db810911899e13dc03745dd0f6c1f8106f1d20cf9c866c885cdaa072b64c1bb92d2f52907f1f4a00a6351d9f57d46be07f353e50b7fe31cd297c02a449a8
+  metadata.gz: 3c64fa145b1717cbf966b1e92052372fec3cd7848e97c128b769aba1721c4d5a852f1a1c7d656017c529009492a3f530c863e328275aab57e2c81f6942f3bd31
+  data.tar.gz: d0f35afc155263d59d5c491784481f189d491b5c5866fc7a983dc3621dc7b7efbf1173524ac9490310c725a926fc44cff89c4299aac8a02be1947bfe156feeb1

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.6.8 - 2018-04-12
+## 0.6.9 - 2018-04-16
 * [fixed] Fix wrapped TimeoutException not retry [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
 * [enhance] Make read_timeout configurable [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.9"
+version = "0.6.10.alpha"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -14,10 +14,6 @@ public interface MarketoService
 {
     List<MarketoField> describeLead();
 
-    List<MarketoField> describeLeadByProgram();
-
-    List<MarketoField> describeLeadByLists();
-
     File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
     File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -185,22 +185,6 @@ public class MarketoServiceImpl implements MarketoService
         return marketoRestClient.describeLead();
     }
 
-    @Override
-    public List<MarketoField> describeLeadByProgram()
-    {
-        List<MarketoField> columns = marketoRestClient.describeLead();
-        columns.add(new MarketoField(MarketoUtils.PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
-        return columns;
-    }
-
-    @Override
-    public List<MarketoField> describeLeadByLists()
-    {
-        List<MarketoField> columns = marketoRestClient.describeLead();
-        columns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
-        return columns;
-    }
-
     private static class DownloadBulkExtractException extends Exception
     {
         private final long byteWritten;

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -7,8 +7,6 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
-import org.embulk.input.marketo.MarketoUtils;
-import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
@@ -29,7 +27,7 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
 
     private static final String UPDATED_AT = "updatedAt";
 
-    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
+    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
     {
         @Config("use_updated_at")
         @ConfigDefault("false")
@@ -63,9 +61,8 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<MarketoField> columns = marketoService.describeLead();
-            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns));
-            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
+            LeadServiceResponseMapperBuilder<PluginTask> leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(task, marketoService);
+            return leadServiceResponseMapperBuilder.buildServiceResponseMapper(task);
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java

@@ -0,0 +1,85 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.ServiceResponseMapperBuildable;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 5/21/18.
+ */
+public class LeadServiceResponseMapperBuilder<T extends LeadServiceResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
+{
+    private static final Logger LOGGER = Exec.getLogger(LeadServiceResponseMapperBuilder.class);
+    private MarketoService marketoService;
+
+    private T pluginTask;
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("included_fields")
+        @ConfigDefault("null")
+        Optional<List<String>> getIncludedLeadFields();
+
+        @Config("extracted_fields")
+        @ConfigDefault("[]")
+        List<String> getExtractedFields();
+
+        void setExtractedFields(List<String> extractedFields);
+    }
+
+    public LeadServiceResponseMapperBuilder(T task, MarketoService marketoService)
+    {
+        this.pluginTask = task;
+        this.marketoService = marketoService;
+    }
+
+    protected List<MarketoField> getLeadColumns()
+    {
+        List<MarketoField> columns = marketoService.describeLead();
+        if (pluginTask.getIncludedLeadFields().isPresent() && !pluginTask.getIncludedLeadFields().get().isEmpty()) {
+            List<MarketoField> filteredColumns = new ArrayList<>();
+            List<String> includedFields = pluginTask.getIncludedLeadFields().get();
+            for (String fieldName : includedFields) {
+                Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
+                if (includedField.isPresent()) {
+                    filteredColumns.add(includedField.get());
+                }
+                else {
+                    LOGGER.warn("Included field [{}] not found in Marketo lead field", fieldName);
+                }
+            }
+            columns = filteredColumns;
+            LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
+        }
+        return columns;
+    }
+
+    private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
+    {
+        for (MarketoField marketoField : inputList) {
+            if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
+                return Optional.of(marketoField);
+            }
+        }
+        return Optional.absent();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
+    {
+        List<MarketoField> leadColumns = getLeadColumns();
+        pluginTask.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(leadColumns));
+        return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), leadColumns);
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java

@@ -10,6 +10,7 @@ import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
@@ -18,7 +19,7 @@ import java.util.List;
  */
 public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithListInputPlugin.PluginTask>
 {
-    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
     {
     }
 
@@ -29,7 +30,10 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
     @Override
     protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
     {
-        return FluentIterable.from(marketoService.getAllListLead(task.getExtractedFields())).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+        List<String> extractedFields = task.getExtractedFields();
+        // Remove LIST_ID_COLUMN_NAME when sent fields to Marketo since LIST_ID_COLUMN_NAME are added by plugin code
+        extractedFields.remove(MarketoUtils.LIST_ID_COLUMN_NAME);
+        return FluentIterable.from(marketoService.getAllListLead(extractedFields)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
     }
 
     @Override
@@ -37,9 +41,25 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<MarketoField> columns = marketoService.describeLeadByLists();
-            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns, MarketoUtils.LIST_ID_COLUMN_NAME));
-            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
+            LeadWithListServiceResponseMapper serviceResponseMapper = new LeadWithListServiceResponseMapper(task, marketoService);
+            return serviceResponseMapper.buildServiceResponseMapper(task);
+        }
+    }
+
+    private static class LeadWithListServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
+    {
+
+        public LeadWithListServiceResponseMapper(LeadWithListInputPlugin.PluginTask task, MarketoService marketoService)
+        {
+            super(task, marketoService);
+        }
+
+        @Override
+        protected List<MarketoField> getLeadColumns()
+        {
+            List<MarketoField> leadColumns = super.getLeadColumns();
+            leadColumns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+            return leadColumns;
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java

@@ -18,7 +18,7 @@ import java.util.List;
  */
 public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithProgramInputPlugin.PluginTask>
 {
-    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
     {
     }
 
@@ -26,6 +26,8 @@ public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<L
     protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
     {
         List<String> fieldNames = task.getExtractedFields();
+        // Remove PROGRAM_ID_COLUMN_NAME when sent fields to Marketo since PROGRAM_ID_COLUMN_NAME are added by plugin code
+        fieldNames.remove(MarketoUtils.PROGRAM_ID_COLUMN_NAME);
         return FluentIterable.from(marketoService.getAllProgramLead(fieldNames)).
                 transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
     }
@@ -35,9 +37,24 @@ public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<L
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<MarketoField> columns = marketoService.describeLeadByProgram();
-            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns, MarketoUtils.PROGRAM_ID_COLUMN_NAME));
-            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
+            LeadWithProgramServiceResponseMapper serviceResponseMapper = new LeadWithProgramServiceResponseMapper(task, marketoService);
+            return serviceResponseMapper.buildServiceResponseMapper(task);
+        }
+    }
+
+    private static class LeadWithProgramServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
+    {
+        public LeadWithProgramServiceResponseMapper(LeadWithProgramInputPlugin.PluginTask task, MarketoService marketoService)
+        {
+            super(task, marketoService);
+        }
+
+        @Override
+        protected List<MarketoField> getLeadColumns()
+        {
+            List<MarketoField> leadColumns = super.getLeadColumns();
+            leadColumns.add(new MarketoField(MarketoUtils.PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+            return leadColumns;
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -45,7 +45,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.Set;
 
 /**
  * Created by tai.khuu on 9/18/17.

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -1,16 +1,14 @@
 package org.embulk.input.marketo.delegate;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
 import org.embulk.base.restclient.DefaultServiceDataSplitter;
 import org.embulk.base.restclient.RestClientInputPluginDelegate;
 import org.embulk.base.restclient.RestClientInputTaskBase;
 import org.embulk.base.restclient.ServiceDataSplitter;
 import org.embulk.base.restclient.record.RecordImporter;
 import org.embulk.base.restclient.record.ServiceRecord;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
+import org.embulk.config.*;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.rest.MarketoRestClient;
@@ -30,33 +28,13 @@ import java.util.List;
 public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputPluginDelegate.PluginTask> implements RestClientInputPluginDelegate<T>
 {
     public static final int PREVIEW_RECORD_LIMIT = 15;
-    private static final int CONNECT_TIMEOUT_IN_MILLIS = 30000;
-    private static final int IDLE_TIMEOUT_IN_MILLIS = 60000;
     public interface PluginTask
             extends RestClientInputTaskBase, MarketoRestClient.PluginTask
     {
-        @Config("maximum_retries")
-        @ConfigDefault("7")
-        Integer getMaximumRetries();
-
-        @Config("initial_retry_interval_milis")
-        @ConfigDefault("20000")
-        Integer getInitialRetryIntervalMilis();
-
-        @Config("maximum_retries_interval_milis")
-        @ConfigDefault("120000")
-        Integer getMaximumRetriesIntervalMilis();
-
         @Config("schema_column_prefix")
         @ConfigDefault("\"mk\"")
         String getSchemaColumnPrefix();
 
-        @Config("extracted_fields")
-        @ConfigDefault("[]")
-        List<String> getExtractedFields();
-
-        void setExtractedFields(List<String> extractedFields);
-
         DateTime getJobStartTime();
 
         void setJobStartTime(DateTime dateTime);
@@ -77,6 +55,9 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
     @Override
     public TaskReport ingestServiceData(T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
     {
+        if (Exec.isPreview()) {
+            task.setBatchSize(PREVIEW_RECORD_LIMIT);
+        }
         try (MarketoRestClient restClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(restClient);
             Iterator<ServiceRecord> serviceRecords = getServiceRecords(marketoService, task);
@@ -95,10 +76,7 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
     @VisibleForTesting
     public MarketoRestClient createMarketoRestClient(PluginTask task)
     {
-        if (Exec.isPreview()) {
-            task.setBatchSize(PREVIEW_RECORD_LIMIT);
-        }
-        return new MarketoRestClient(task, new Jetty92RetryHelper(task.getMaximumRetries(), task.getInitialRetryIntervalMilis(), task.getMaximumRetriesIntervalMilis(), new DefaultJetty92ClientCreator(CONNECT_TIMEOUT_IN_MILLIS, IDLE_TIMEOUT_IN_MILLIS)));
+        return new MarketoRestClient(task);
     }
 
     @Override

data/src/main/java/org/embulk/input/marketo/model/MarketoField.java

@@ -114,4 +114,13 @@ public class MarketoField
             return Optional.fromNullable(format);
         }
     }
+
+    @Override
+    public String toString()
+    {
+        return "MarketoField{" +
+                "name='" + name + '\'' +
+                ", marketoDataType=" + marketoDataType +
+                '}';
+    }
 }

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -23,6 +23,7 @@ import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
+import org.embulk.util.retryhelper.jetty92.DefaultJetty92ClientCreator;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.slf4j.Logger;
 
@@ -53,6 +54,9 @@ public class MarketoRestClient extends MarketoBaseRestClient
 
     private static final String RANGE_HEADER = "Range";
 
+    private static final int CONNECT_TIMEOUT_IN_MILLIS = 30000;
+    private static final int IDLE_TIMEOUT_IN_MILLIS = 60000;
+
     private String endPoint;
 
     private Integer batchSize;
@@ -108,11 +112,23 @@ public class MarketoRestClient extends MarketoBaseRestClient
         @Config("read_timeout_millis")
         @ConfigDefault("60000")
         Long getReadTimeoutMillis();
+
+        @Config("maximum_retries")
+        @ConfigDefault("7")
+        Integer getMaximumRetries();
+
+        @Config("initial_retry_interval_milis")
+        @ConfigDefault("20000")
+        Integer getInitialRetryIntervalMilis();
+
+        @Config("maximum_retries_interval_milis")
+        @ConfigDefault("120000")
+        Integer getMaximumRetriesIntervalMilis();
     }
 
-    public MarketoRestClient(PluginTask task, Jetty92RetryHelper retryHelper)
+    public MarketoRestClient(PluginTask task)
     {
-        this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getReadTimeoutMillis(), task.getMarketoLimitIntervalMilis(), retryHelper);
+        this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getReadTimeoutMillis(), task.getMarketoLimitIntervalMilis(), new Jetty92RetryHelper(task.getMaximumRetries(), task.getInitialRetryIntervalMilis(), task.getMaximumRetriesIntervalMilis(), new DefaultJetty92ClientCreator(CONNECT_TIMEOUT_IN_MILLIS, IDLE_TIMEOUT_IN_MILLIS)));
     }
 
     public MarketoRestClient(String endPoint, String identityEndPoint, String clientId, String clientSecret, Integer batchSize, Integer maxReturn, long readTimeoutMilis, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)

data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java

@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.io.ByteStreams;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.input.marketo.model.BulkExtractRangeHeader;
-import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.input.marketo.rest.RecordPagingIterable;
 import org.junit.Assert;
@@ -135,28 +134,4 @@ public class MarketoServiceImplTest
         marketoService.describeLead();
         Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
     }
-
-    @Test
-    public void describeLeadByProgram() throws Exception
-    {
-        List<MarketoField> marketoFields = new ArrayList<>();
-        Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
-        marketoService.describeLeadByProgram();
-        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
-        Assert.assertEquals(1, marketoFields.size());
-        Assert.assertEquals("programId", marketoFields.get(0).getName());
-        Assert.assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
-    }
-
-    @Test
-    public void describeLeadByLists() throws Exception
-    {
-        List<MarketoField> marketoFields = new ArrayList<>();
-        Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
-        marketoService.describeLeadByLists();
-        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
-        Assert.assertEquals(1, marketoFields.size());
-        Assert.assertEquals("listId", marketoFields.get(0).getName());
-        Assert.assertEquals(MarketoField.MarketoDataType.STRING, marketoFields.get(0).getMarketoDataType());
-    }
 }

data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java

@@ -0,0 +1,119 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Schema;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 5/24/18.
+ */
+public class LeadServiceResponseMapperBuilderTest
+{
+    private LeadServiceResponseMapperBuilder<LeadServiceResponseMapperBuilder.PluginTask> leadServiceResponseMapperBuilder;
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private LeadServiceResponseMapperBuilder.PluginTask pluginTask;
+
+    private ConfigSource configSource;
+
+    private MarketoService marketoService;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        ConfigLoader configLoader = embulkTestRuntime.getExec().getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYamlString("--- \n" +
+                "account_id: 389-FLQ-873\n" +
+                "client_id: 87bdd058-12ff-49b8-8d29-c0518669e59a\n" +
+                "client_secret: 2uauyk43zzwjlUZwboZVxQ3SMTHe8ILY\n" +
+                "included_fields: \n" +
+                "  - company\n" +
+                "  - site\n" +
+                "  - billingstreet\n" +
+                "  - billingcity\n" +
+                "  - billingstate\n" +
+                "  - billingcountry\n" +
+                "  - billingpostalCode\n" +
+                "  - website\n" +
+                "  - mainphone\n" +
+                "  - annualrevenue\n" +
+                "  - numberofemployees\n" +
+                "  - industry\n" +
+                "  - siccode\n" +
+                "  - mktoCompanyNotes\n" +
+                "  - externalcompanyId\n" +
+                "  - id\n" +
+                "target: all_lead_with_list_id\n");
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        marketoService = Mockito.mock(MarketoService.class);
+        JavaType marketoFieldsType = MarketoUtils.OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = MarketoUtils.OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
+        Mockito.when(marketoService.describeLead()).thenReturn(marketoFields);
+    }
+
+    @Test
+    public void buildServiceResponseMapper() throws Exception
+    {
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        ServiceResponseMapper<? extends ValueLocator> serviceResponseMapper = leadServiceResponseMapperBuilder.buildServiceResponseMapper(pluginTask);
+        Assert.assertFalse(pluginTask.getExtractedFields().isEmpty());
+        Assert.assertEquals(16, pluginTask.getExtractedFields().size());
+        Schema embulkSchema = serviceResponseMapper.getEmbulkSchema();
+        Assert.assertEquals(16, embulkSchema.getColumns().size());
+        Assert.assertEquals("mk_billingStreet", embulkSchema.getColumns().get(2).getName());
+        Assert.assertEquals("mk_company", embulkSchema.getColumn(0).getName());
+        Assert.assertEquals("mk_billingStreet", embulkSchema.getColumn(2).getName());
+        Assert.assertEquals("mk_id", embulkSchema.getColumn(15).getName());
+    }
+
+    @Test
+    public void getLeadColumnsIncludedEmpty() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertEquals(129, leadColumns.size());
+    }
+
+    @Test
+    public void getLeadColumnsIncluded1() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[\"company\",\"incorrect_value\"]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertEquals(1, leadColumns.size());
+        Assert.assertEquals("company", leadColumns.get(0).getName());
+    }
+
+    @Test
+    public void getLeadColumnsIncluded2() throws IOException
+    {
+        configSource = configSource.set("included_fields", MarketoUtils.OBJECT_MAPPER.readTree("[\"company\",\"incorrect_value\"]"));
+        pluginTask = configSource.loadConfig(LeadServiceResponseMapperBuilder.PluginTask.class);
+        marketoService = Mockito.mock(MarketoService.class);
+        leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(pluginTask, marketoService);
+        Mockito.when(marketoService.describeLead()).thenReturn(new ArrayList<MarketoField>());
+        List<MarketoField> leadColumns = leadServiceResponseMapperBuilder.getLeadColumns();
+        Assert.assertTrue(leadColumns.isEmpty());
+    }
+}

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -72,7 +72,7 @@ public class MarketoRestClientTest
         configSource.set("max_return", 2);
         MarketoRestClient.PluginTask task = configSource.loadConfig(MarketoRestClient.PluginTask.class);
         mockRetryHelper = Mockito.mock(Jetty92RetryHelper.class);
-        MarketoRestClient realRestClient = new MarketoRestClient(task, mockRetryHelper);
+        MarketoRestClient realRestClient = new MarketoRestClient(task);
         marketoRestClient = Mockito.spy(realRestClient);
     }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.9
+  version: 0.6.10.alpha
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-16 00:00:00.000000000 Z
+date: 2018-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,6 +73,7 @@ files:
 - src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java
+- src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java
@@ -97,6 +98,7 @@ files:
 - src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java
+- src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java
@@ -124,9 +126,9 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-input-marketo-0.6.10.alpha.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.9.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -144,9 +146,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9