embulk-input-marketo 0.6.11 → 0.6.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b375807158e07d67abc81e6319ef6ec9a087f9b
-  data.tar.gz: 596f26cb4804ce01819b27499ead25b35c567100
+  metadata.gz: edba80ca03303c028b5ca8a2d70f7a7c801ec844
+  data.tar.gz: 4d935d696fb3b8adca5a0ac3c44684378f3b67a0
 SHA512:
-  metadata.gz: d5743056090cbad09a2a75a6f460f2d843e721e254b2e898e7e809faad8f0309732244282c91f5dd6bbed9cafb95d0be519bba6305e8bee2726e49b0905867f8
-  data.tar.gz: ab9fe2a479f4ccc842efb62e80d14dc4bac8941b64dc6c7d589c35a412a6f782fc902edd75a014e48b410889dca3e63c99a184dd533675c0707c44f520da0696
+  metadata.gz: 20c5faedf4785d9fc4bfc4f296a851a5b1c009b470ec91a86a870517b73a9333caf85c9a7e892a6d209cd9b52fb387d09b2b02937bb0cfa6a1676f41114a42b4
+  data.tar.gz: 8fa3f62aec0903509f2723cb08e1fb1af5b8dffd196fa6f3e5c519085cffe0fca726f453d730f021ba00a3ebed3c4ea8dadcaaa2846932b26c8743443e88f8e5

data/.gitignore

@@ -12,4 +12,3 @@ build/
 .classpath
 .project
 
-/bin/

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.12 - 2018-11-09
+* [enhance] Implement Custom Object [#90](https://github.com/treasure-data/embulk-input-marketo/pull/90)
+
 ## 0.6.11 - 2018-09-10
 * [enhance] Implement Assets Programs [#89](https://github.com/treasure-data/embulk-input-marketo/pull/89)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.11"
+version = "0.6.12"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -11,6 +11,7 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
 import org.embulk.input.marketo.delegate.ActivityBulkExtractInputPlugin;
 import org.embulk.input.marketo.delegate.CampaignInputPlugin;
+import org.embulk.input.marketo.delegate.CustomObjectInputPlugin;
 import org.embulk.input.marketo.delegate.LeadBulkExtractInputPlugin;
 import org.embulk.input.marketo.delegate.LeadWithListInputPlugin;
 import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin;
@@ -29,7 +30,8 @@ public class MarketoInputPluginDelegate
             ActivityBulkExtractInputPlugin.PluginTask,
             CampaignInputPlugin.PluginTask,
             ProgramInputPlugin.PluginTask,
-            MarketoRestClient.PluginTask
+            MarketoRestClient.PluginTask,
+            CustomObjectInputPlugin.PluginTask
     {
         @Config("target")
         Target getTarget();
@@ -73,7 +75,8 @@ public class MarketoInputPluginDelegate
         CAMPAIGN(new CampaignInputPlugin()),
         ALL_LEAD_WITH_LIST_ID(new LeadWithListInputPlugin()),
         ALL_LEAD_WITH_PROGRAM_ID(new LeadWithProgramInputPlugin()),
-        PROGRAM(new ProgramInputPlugin());
+        PROGRAM(new ProgramInputPlugin()),
+        CUSTOM_OBJECT(new CustomObjectInputPlugin());
 
         private RestClientInputPluginDelegate restClientInputPluginDelegate;
 

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -29,4 +29,8 @@ public interface MarketoService
     Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue);
 
     Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues);
+
+    Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
+
+    List<MarketoField> describeCustomObject(String customObjectAPIName);
 }

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -224,4 +224,16 @@ public class MarketoServiceImpl implements MarketoService
     {
         return marketoRestClient.getProgramsByDateRange(earliestUpdatedAt, latestUpdatedAt, filterType, filterValues);
     }
+
+    @Override
+    public List<MarketoField> describeCustomObject(String customObjectAPIName)
+    {
+        return marketoRestClient.describeCustomObject(customObjectAPIName);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue)
+    {
+        return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
+    }
 }

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java

@@ -0,0 +1,75 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+import java.util.Iterator;
+
+public class CustomObjectInputPlugin extends MarketoBaseInputPluginDelegate<CustomObjectInputPlugin.PluginTask>
+{
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CustomObjectResponseMapperBuilder.PluginTask
+    {
+        @Config("custom_object_filter_type")
+        @ConfigDefault("\"\"")
+        String getCustomObjectFilterType();
+
+        @Config("custom_object_filter_from_value")
+        @ConfigDefault("1")
+        Integer getFromValue();
+
+        @Config("custom_object_filter_to_value")
+        @ConfigDefault("null")
+        Optional<Integer> getToValue();
+    }
+
+    public CustomObjectInputPlugin()
+    {
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        super.validateInputTask(task);
+        if (StringUtils.isBlank(task.getCustomObjectFilterType())) {
+            throw new ConfigException("`custom_object_filter_type` cannot be empty");
+        }
+        if (StringUtils.isBlank(task.getCustomObjectAPIName())) {
+            throw new ConfigException("`custom_object_api_name` cannot be empty");
+        }
+        if (task.getToValue().isPresent() && task.getToValue().get() < task.getFromValue()) {
+            throw new ConfigException(String.format("`to_value` (%s) cannot be less than  the `from_value` (%s)", task.getToValue().get(), task.getFromValue()));
+        }
+    }
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        return FluentIterable
+                .from(marketoService.getCustomObject(task.getCustomObjectAPIName(), task.getCustomObjectFilterType(), task.getCustomObjectFields().orNull(), task.getFromValue(), task.getToValue().orNull()))
+                .transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+            CustomObjectResponseMapperBuilder<PluginTask> customObjectResponseMapperBuilder = new CustomObjectResponseMapperBuilder<>(task, marketoService);
+            return customObjectResponseMapperBuilder.buildServiceResponseMapper(task);
+        }
+    }
+
+}

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java

@@ -0,0 +1,80 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.ServiceResponseMapperBuildable;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class CustomObjectResponseMapperBuilder<T extends CustomObjectResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
+{
+    private static final Logger LOGGER = Exec.getLogger(CustomObjectResponseMapperBuilder.class);
+    private MarketoService marketoService;
+
+    private T pluginTask;
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("custom_object_api_name")
+        @ConfigDefault("\"\"")
+        String getCustomObjectAPIName();
+
+        @Config("custom_object_fields")
+        @ConfigDefault("null")
+        Optional<String> getCustomObjectFields();
+    }
+
+    public CustomObjectResponseMapperBuilder(T task, MarketoService marketoService)
+    {
+        this.pluginTask = task;
+        this.marketoService = marketoService;
+    }
+
+    protected List<MarketoField> getCustomObjectColumns()
+    {
+        List<MarketoField> columns = marketoService.describeCustomObject(pluginTask.getCustomObjectAPIName());
+        if (pluginTask.getCustomObjectFields().isPresent() && StringUtils.isNotBlank(pluginTask.getCustomObjectFields().get())) {
+            List<MarketoField> filteredColumns = new ArrayList<>();
+            List<String> includedFields = Arrays.asList(pluginTask.getCustomObjectFields().get().split(","));
+            for (String fieldName : includedFields) {
+                Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
+                if (includedField.isPresent()) {
+                    filteredColumns.add(includedField.get());
+                }
+                else {
+                    LOGGER.warn("Included field [{}] not found in Marketo Custom Object [{}] field", fieldName, pluginTask.getCustomObjectAPIName());
+                }
+            }
+            columns = filteredColumns;
+            LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
+        }
+        return columns;
+    }
+
+    private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
+    {
+        for (MarketoField marketoField : inputList) {
+            if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
+                return Optional.of(marketoField);
+            }
+        }
+        return Optional.absent();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
+    {
+        List<MarketoField> customObjectColumns = getCustomObjectColumns();
+        return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), customObjectColumns);
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java

@@ -24,7 +24,9 @@ public enum MarketoRESTEndpoint
     GET_PROGRAMS("/rest/asset/v1/programs.json"),
     GET_LEADS_BY_PROGRAM("/rest/v1/leads/programs/${program_id}.json"),
     GET_CAMPAIGN("/rest/v1/campaigns.json"),
-    GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json");
+    GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json"),
+    GET_CUSTOM_OBJECT("/rest/v1/customobjects/${api_name}.json"),
+    GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json");
 
     private String endpoint;
 

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -1,16 +1,19 @@
 package org.embulk.input.marketo.rest;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Multimap;
 
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jetty.client.util.FormContentProvider;
 import org.eclipse.jetty.util.Fields;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.Task;
 import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.BulkExtractRangeHeader;
@@ -55,6 +58,14 @@ public class MarketoRestClient extends MarketoBaseRestClient
 
     private static final String RANGE_HEADER = "Range";
 
+    private static final String FILTER_TYPE = "filterType";
+
+    private static final String FILTER_VALUES = "filterValues";
+
+    private static final String FIELDS = "fields";
+
+    private static final int MAX_REQUEST_SIZE = 300;
+
     private static final int CONNECT_TIMEOUT_IN_MILLIS = 30000;
     private static final int IDLE_TIMEOUT_IN_MILLIS = 60000;
 
@@ -435,4 +446,92 @@ public class MarketoRestClient extends MarketoBaseRestClient
         }
         return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), multimap, ObjectNode.class);
     }
+
+    public List<MarketoField> describeCustomObject(String apiName)
+    {
+        MarketoResponse<ObjectNode> jsonResponse = doGet(endPoint + MarketoRESTEndpoint.GET_CUSTOM_OBJECT_DESCRIBE.getEndpoint(new ImmutableMap.Builder().put("api_name", apiName).build()), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(this.readTimeoutMillis));
+        if (jsonResponse.getResult().size() == 0) {
+            throw new ConfigException(String.format("Custom Object %s is not exits.", apiName));
+        }
+        List<MarketoField> marketoFields = new ArrayList<>();
+        JsonNode fieldNodes = jsonResponse.getResult().get(0).path("fields");
+        for (JsonNode node : fieldNodes) {
+            String dataType = node.get("dataType").asText();
+            String name = node.get("name").asText();
+            marketoFields.add(new MarketoField(name, dataType));
+        }
+        if (marketoFields.size() == 0) {
+            throw new ConfigException(String.format("Custom Object %s don't have any field data.", apiName));
+        }
+        return marketoFields;
+    }
+    private <T> RecordPagingIterable<T> getCustomObjectRecordWithPagination(final String endPoint, final String customObjectFilterType, final String customObjectFields, final Integer fromValue, final Integer toValue, final Class<T> recordClass)
+    {
+        return new RecordPagingIterable<>(new RecordPagingIterable.PagingFunction<RecordPagingIterable.OffsetWithTokenPage<T>>()
+        {
+            @Override
+            public RecordPagingIterable.OffsetWithTokenPage<T> getNextPage(RecordPagingIterable.OffsetWithTokenPage<T> currentPage)
+            {
+                return getOffsetPage(currentPage.getNextOffSet(), currentPage.getNextPageToken());
+            }
+
+            @Override
+            public RecordPagingIterable.OffsetWithTokenPage<T> getFirstPage()
+            {
+                return getOffsetPage(fromValue, "");
+            }
+
+            private RecordPagingIterable.OffsetWithTokenPage<T> getOffsetPage(int offset, String nextPageToken)
+            {
+                boolean isMoreResult = true;
+                boolean isEndOffset = false;
+                int nextOffset = offset + MAX_REQUEST_SIZE;
+
+                if (toValue != null) {
+                    if (toValue <= nextOffset) {
+                        nextOffset = toValue + 1;
+                        isEndOffset = true;
+                    }
+                }
+                StringBuilder filterValues = new StringBuilder();
+                for (int i = offset; i < (nextOffset - 1); i++) {
+                    filterValues.append(String.valueOf(i)).append(",");
+                }
+                filterValues.append(String.valueOf(nextOffset - 1));
+
+                ImmutableListMultimap.Builder<String, String> params = new ImmutableListMultimap.Builder<>();
+                params.put(FILTER_TYPE, customObjectFilterType);
+                params.put(FILTER_VALUES, filterValues.toString());
+                if (StringUtils.isNotBlank(nextPageToken)) {
+                    params.put(NEXT_PAGE_TOKEN, nextPageToken);
+                }
+                if (customObjectFields != null) {
+                    params.put(FIELDS, customObjectFields);
+                }
+                MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(readTimeoutMillis, recordClass));
+                String nextToken = "";
+                if (StringUtils.isNotBlank(marketoResponse.getNextPageToken())) {
+                    nextToken = marketoResponse.getNextPageToken();
+                    //skip offset when nextPageToken is exits.
+                    nextOffset = offset;
+                }
+
+                if (toValue == null) {
+                    if (marketoResponse.getResult().isEmpty()) {
+                        isMoreResult = false;
+                    }
+                }
+                else {
+                    if (isEndOffset && StringUtils.isBlank(nextToken)) {
+                        isMoreResult = false;
+                    }
+                }
+                return new RecordPagingIterable.OffsetWithTokenPage<>(marketoResponse.getResult(), nextOffset, nextToken, isMoreResult);
+            }
+        });
+    }
+    public Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue)
+    {
+        return getCustomObjectRecordWithPagination(endPoint + MarketoRESTEndpoint.GET_CUSTOM_OBJECT.getEndpoint(new ImmutableMap.Builder().put("api_name", customObjectAPIName).build()), customObjectFilterType, customObjectFields, fromValue, toValue, ObjectNode.class);
+    }
 }

data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java

@@ -151,4 +151,30 @@ public class RecordPagingIterable<T> implements Iterable<T>
             return nextOffSet;
         }
     }
+    public static class OffsetWithTokenPage<T> extends Page<T>
+    {
+        private int nextOffSet;
+        private String nextPageToken;
+        public OffsetWithTokenPage(Iterable<T> records, int nextOffSet, String nextPageToken, boolean moreResult)
+        {
+            super(records, moreResult);
+            this.nextOffSet = nextOffSet;
+            this.nextPageToken = nextPageToken;
+        }
+
+        public int getNextOffSet()
+        {
+            return nextOffSet;
+        }
+
+        public String getNextPageToken()
+        {
+            return nextPageToken;
+        }
+
+        public void setNextPageToken(String nextPageToken)
+        {
+            this.nextPageToken = nextPageToken;
+        }
+    }
 }

data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java

@@ -0,0 +1,101 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+
+public class CustomObjectInputPluginTest
+{
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    private CustomObjectInputPlugin customObjectInputPlugin;
+
+    private ConfigSource configSource;
+
+    private MarketoRestClient mockMarketoRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        customObjectInputPlugin = Mockito.spy(new CustomObjectInputPlugin());
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/custom_object_config.yaml"));
+        mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockMarketoRestClient).when(customObjectInputPlugin).createMarketoRestClient(Mockito.any(CustomObjectInputPlugin.PluginTask.class));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void validateCustomObjectFilterTypeError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = Mockito.mock(CustomObjectInputPlugin.PluginTask.class);
+        Mockito.when(pluginTask.getCustomObjectFilterType()).thenReturn("");
+        customObjectInputPlugin.validateInputTask(pluginTask);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void validateCustomObjectAPINameError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = Mockito.mock(CustomObjectInputPlugin.PluginTask.class);
+        Mockito.when(pluginTask.getCustomObjectAPIName()).thenReturn("");
+        customObjectInputPlugin.validateInputTask(pluginTask);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void validateFromValueGreaterThanToValueError()
+    {
+        CustomObjectInputPlugin.PluginTask pluginTask = Mockito.mock(CustomObjectInputPlugin.PluginTask.class);
+        Mockito.when(pluginTask.getFromValue()).thenReturn(100);
+        Mockito.when(pluginTask.getToValue()).thenReturn(Optional.of(90));
+        customObjectInputPlugin.validateInputTask(pluginTask);
+    }
+
+    @Test
+    public void testRun() throws IOException
+    {
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/custom_object_response_full.json"), javaType);
+        JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/custom_object_describe_marketo_fields_full.json"), marketoFieldsType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+        Mockito.when(mockMarketoRestClient.getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())).thenReturn(mockRecordPagingIterable);
+        Mockito.when(mockMarketoRestClient.describeCustomObject(Mockito.anyString())).thenReturn(marketoFields);
+        CustomObjectInputPlugin.PluginTask task = configSource.loadConfig(CustomObjectInputPlugin.PluginTask.class);
+        ServiceResponseMapper<? extends ValueLocator> mapper = customObjectInputPlugin.buildServiceResponseMapper(task);
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        customObjectInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCustomObject(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt());
+        Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeCustomObject(Mockito.anyString());
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Mockito.verify(mockPageBuilder, Mockito.times(3)).setLong(Mockito.eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        assertArrayEquals(new Long[]{1L, 2L, 3L}, allValues.toArray());
+    }
+}

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -544,4 +544,39 @@ public class MarketoRestClientTest
         Assert.assertEquals("filter1", params2.get("filterType").get(0));
         Assert.assertEquals(String.join(",", filterValues), params2.get("filterValues").get(0));
     }
+
+    @Test
+    public void describeCustomObject() throws Exception
+    {
+        String customObjectSchema = new String(ByteStreams.toByteArray(this.getClass().getResourceAsStream("/fixtures/custom_object_describe.json")));
+        MarketoResponse<ObjectNode> marketoResponse = OBJECT_MAPPER.readValue(customObjectSchema, RESPONSE_TYPE);
+        String apiName = "custom_object";
+        Map<String, String> pathParams = new HashMap<>();
+        pathParams.put("api_name", apiName);
+        Mockito.doReturn(marketoResponse).when(marketoRestClient).doGet(Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_CUSTOM_OBJECT_DESCRIBE.getEndpoint(pathParams)), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.any(MarketoResponseJetty92EntityReader.class));
+        List<MarketoField> marketoFields = marketoRestClient.describeCustomObject(apiName);
+        Assert.assertEquals(16, marketoFields.size());
+        JavaType marketoFieldType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
+        List<MarketoField> expectedFields = OBJECT_MAPPER.readValue(new String(ByteStreams.toByteArray(this.getClass().getResourceAsStream("/fixtures/custom_object_expected.json"))), marketoFieldType);
+        Assert.assertArrayEquals(expectedFields.toArray(), marketoFields.toArray());
+    }
+
+    @Test
+    public void getCustomObject() throws Exception
+    {
+        String apiName = "custom_object";
+        Map<String, String> pathParams = new HashMap<>();
+        pathParams.put("api_name", apiName);
+
+        ArrayNode listPages = (ArrayNode) OBJECT_MAPPER.readTree(new String(ByteStreams.toByteArray(this.getClass().getResourceAsStream("/fixtures/custom_object_response.json")))).get("responses");
+        MarketoResponse<ObjectNode> page1 = OBJECT_MAPPER.readValue(listPages.get(0).toString(), RESPONSE_TYPE);
+        Mockito.doReturn(page1).when(marketoRestClient).doGet(Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_CUSTOM_OBJECT.getEndpoint(pathParams)), Mockito.isNull(Map.class), Mockito.any(ImmutableListMultimap.class), Mockito.any(MarketoResponseJetty92EntityReader.class));
+        RecordPagingIterable<ObjectNode> pages = (RecordPagingIterable<ObjectNode>) marketoRestClient.getCustomObject(apiName, "id", null, 1, 2);
+        Iterator<ObjectNode> iterator = pages.iterator();
+        ObjectNode customObject1 = iterator.next();
+        ObjectNode customObject2 = iterator.next();
+        Assert.assertFalse(iterator.hasNext());
+        Assert.assertEquals("1", customObject1.get("id").asText());
+        Assert.assertEquals("2", customObject2.get("id").asText());
+    }
 }

data/src/test/resources/config/custom_object_config.yaml

@@ -0,0 +1,8 @@
+account_id: 'account_id'
+client_id: client_id
+client_secret: client_secret
+custom_object_api_name: custom_object_api_name
+custom_object_filter_type: custom_object_filter_type
+custom_object_fields: ""
+custom_object_filter_from_value: 1
+custom_object_filter_to_value: 2

data/src/test/resources/fixtures/custom_object_describe.json

@@ -0,0 +1,124 @@
+{
+  "requestId":"185d6#14b51985ff0",
+  "success":true,
+  "result":[
+    {
+      "name":"custom_object",
+      "displayName":"Custom Object",
+      "description":"Custom Object",
+      "createdAt":"2015-02-03T22:36:23Z",
+      "updatedAt":"2015-02-03T22:36:24Z",
+      "idField":"marketoGUID",
+      "dedupeFields":["mKtestcurrency"],
+      "searchableFields":[
+        ["mKtestcurrency"],
+        ["marketoGUID"]
+      ],
+      "fields":[
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_boolean",
+          "dataType": "boolean",
+          "name": "mKtestboolean"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_currency",
+          "dataType": "currency",
+          "length": 255,
+          "name": "mKtestcurrency"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_date",
+          "dataType": "date",
+          "name": "mKtestdate"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_Datetime",
+          "dataType": "datetime",
+          "name": "mKtestDatetime"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_email",
+          "dataType": "email",
+          "length": 255,
+          "name": "mKtestemail"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_float",
+          "dataType": "float",
+          "name": "mKtestfloat"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_integer",
+          "dataType": "integer",
+          "name": "mKtestinteger"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_percent",
+          "dataType": "percent",
+          "name": "mKtestpercent"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_phone",
+          "dataType": "phone",
+          "length": 255,
+          "name": "mKtestphone"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_score",
+          "dataType": "score",
+          "name": "mKtestscore"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_string",
+          "dataType": "string",
+          "length": 255,
+          "name": "mKteststring"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_textarea",
+          "dataType": "textarea",
+          "name": "mKtesttextarea"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_url",
+          "dataType": "url",
+          "length": 255,
+          "name": "mKtesturl"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_reference",
+          "dataType": "reference",
+          "length": 255,
+          "name": "mKTestReference"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_formula",
+          "dataType": "formula",
+          "length": 255,
+          "name": "mKTestFormula"
+        },
+        {
+          "updateable":true,
+          "displayName": "Marketo_test_text",
+          "dataType": "text",
+          "name": "mKtesttext"
+        }
+      ]
+    }
+  ]
+}

data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json

@@ -0,0 +1,22 @@
+[
+  {
+    "name": "seq",
+    "marketoDataType": "INTEGER"
+  },
+  {
+    "name": "marketoGUID",
+    "marketoDataType": "STRING"
+  },
+  {
+    "name": "id",
+    "marketoDataType": "INTEGER"
+  },
+  {
+    "name": "createdAt",
+    "marketoDataType": "DATETIME"
+  },
+  {
+    "name": "updatedAt",
+    "marketoDataType": "DATETIME"
+  }
+]

data/src/test/resources/fixtures/custom_object_expected.json

@@ -0,0 +1,66 @@
+[
+  {
+    "name": "mKtestboolean",
+    "marketoDataType": "BOOLEAN"
+  },
+  {
+    "name": "mKtestcurrency",
+    "marketoDataType": "CURRENCY"
+  },
+  {
+    "name": "mKtestdate",
+    "marketoDataType": "DATE"
+  },
+  {
+    "name": "mKtestDatetime",
+    "marketoDataType": "DATETIME"
+  },
+  {
+    "name": "mKtestemail",
+    "marketoDataType": "EMAIL"
+  },
+  {
+    "name": "mKtestfloat",
+    "marketoDataType": "FLOAT"
+  },
+  {
+    "name": "mKtestinteger",
+    "marketoDataType": "INTEGER"
+  },
+  {
+    "name": "mKtestpercent",
+    "marketoDataType": "PERCENT"
+  },
+  {
+    "name": "mKtestphone",
+    "marketoDataType": "PHONE"
+  },
+  {
+    "name": "mKtestscore",
+    "marketoDataType": "SCORE"
+  },
+  {
+    "name": "mKteststring",
+    "marketoDataType": "STRING"
+  },
+  {
+    "name": "mKtesttextarea",
+    "marketoDataType": "TEXTAREA"
+  },
+  {
+    "name": "mKtesturl",
+    "marketoDataType": "URL"
+  },
+  {
+    "name": "mKTestReference",
+    "marketoDataType": "REFERENCE"
+  },
+  {
+    "name": "mKTestFormula",
+    "marketoDataType": "FORMULA"
+  },
+  {
+    "name": "mKtesttext",
+    "marketoDataType": "TEXT"
+  }
+]

data/src/test/resources/fixtures/custom_object_response.json

@@ -0,0 +1,24 @@
+{
+  "responses":[
+    {
+      "requestId": "83e9#166f835a592",
+      "result": [
+        {
+          "seq": 0,
+          "marketoGUID": "aa168cfa-626d-49b4-b735-526e0b8c069e",
+          "id": 1,
+          "createdAt": "2018-11-06T12:01:49Z",
+          "updatedAt": "2018-11-06T12:01:49Z"
+        },
+        {
+          "seq": 1,
+          "marketoGUID": "da7c347d-754b-44f3-9821-d1233f0852f6",
+          "id": 2,
+          "createdAt": "2018-11-06T12:01:49Z",
+          "updatedAt": "2018-11-06T12:01:49Z"
+        }
+      ],
+      "success": true
+    }
+  ]
+}

data/src/test/resources/fixtures/custom_object_response_full.json

@@ -0,0 +1,23 @@
+[
+  {
+    "seq": 0,
+    "marketoGUID": "aa168cfa-626d-49b4-b735-526e0b8c069e",
+    "id": 1,
+    "createdAt": "2018-11-06T12:01:49Z",
+    "updatedAt": "2018-11-06T12:01:49Z"
+  },
+  {
+    "seq": 1,
+    "marketoGUID": "da7c347d-754b-44f3-9821-d1233f0852f6",
+    "id": 2,
+    "createdAt": "2018-11-06T12:01:49Z",
+    "updatedAt": "2018-11-06T12:01:49Z"
+  },
+  {
+    "seq": 2,
+    "marketoGUID": "sdfdc347d-754b-5645-9821-d1233f065456",
+    "id": 3,
+    "createdAt": "2018-11-06T12:01:49Z",
+    "updatedAt": "2018-11-06T12:01:49Z"
+  }
+]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.11
+  version: 0.6.12
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-10 00:00:00.000000000 Z
+date: 2018-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,8 @@ files:
 - src/main/java/org/embulk/input/marketo/MarketoUtils.java
 - src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java
+- src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java
+- src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java
 - src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java
@@ -98,6 +100,7 @@ files:
 - src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java
 - src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java
+- src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java
@@ -107,6 +110,7 @@ files:
 - src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
 - src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java
 - src/test/resources/config/activity_bulk_extract_config.yaml
+- src/test/resources/config/custom_object_config.yaml
 - src/test/resources/config/lead_bulk_extract_config.yaml
 - src/test/resources/config/rest_config.yaml
 - src/test/resources/fixtures/activity_extract1.csv
@@ -114,6 +118,11 @@ files:
 - src/test/resources/fixtures/all_program_full.json
 - src/test/resources/fixtures/campaign_response.json
 - src/test/resources/fixtures/campaign_response_full.json
+- src/test/resources/fixtures/custom_object_describe.json
+- src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json
+- src/test/resources/fixtures/custom_object_expected.json
+- src/test/resources/fixtures/custom_object_response.json
+- src/test/resources/fixtures/custom_object_response_full.json
 - src/test/resources/fixtures/lead_by_list.json
 - src/test/resources/fixtures/lead_by_program_response.json
 - src/test/resources/fixtures/lead_describe.json
@@ -130,8 +139,8 @@ files:
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.12.jar
 - classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-input-marketo-0.6.11.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
 licenses: