embulk-input-marketo 0.6.15 → 0.6.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1ecb21401432881f5191869093f9e26d7eecd77
-  data.tar.gz: 80bce389f5f836747a900ce87d2d637125df6f0b
+  metadata.gz: cde8656ca8356a00c7e23c93784ad972b11aafc7
+  data.tar.gz: f2d80312b4daed3022855c36a3101a41c107f52d
 SHA512:
-  metadata.gz: 290ecc3ed678f765358615a0fb2e71bc61142999cf5cae9afa1c4aa60619df2e37778a884201f1505c478aef1d29311afd460f16d705785fc71a4f46f3f9c50c
-  data.tar.gz: 03e2c3625ade8587e3c1730f8ce3a6ffb3baa77195c7a26030fe988cdd22486150862e759c752f27c91ec58cdb20fb2b76441ff46f289936b38128d4b2de202c
+  metadata.gz: 626137ff8225c043183e3739a4b29d2f085b722ec5569c64b172c7b9ce57798a759abef70e0f207bb4bb727a252587ca2bc199e3c21dc3f640bd0d1248607dff
+  data.tar.gz: a03624d1a98621cb4f32e0fcda6b2fe6a29f45879af7324f91e4dd09a3670e5fe0ab89cf5e83a0c9ee500ae58c828ff0bf9779fce9515cc813836c97cfc53303

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.16 - 2019-12-03
+* [enhancement] Added support for ActivityTypeIds filter PR [#96](https://github.com/treasure-data/embulk-input-marketo/pull/96)
+
 ## 0.6.15 - 2019-09-20
 * [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
 

data/README.md

@@ -95,6 +95,8 @@ Incremental support: yes
 
 Range ingestion: yes
 
+Filter by specific activity type ids: yes. See [#95](https://github.com/treasure-data/embulk-input-marketo/issues/95)
+
 ### Campaign
 
 Campaign extract all campaign data from Marketo

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.15"
+version = "0.6.16"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -2,6 +2,7 @@ package org.embulk.input.marketo;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.model.MarketoResponse;
 
 import java.io.File;
 import java.util.Date;
@@ -16,7 +17,7 @@ public interface MarketoService
 
     File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
-    File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
     Iterable<ObjectNode> getAllListLead(List<String> extractFields);
 
@@ -33,4 +34,6 @@ public interface MarketoService
     Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
 
     List<MarketoField> describeCustomObject(String customObjectAPIName);
+
+    Iterable<ObjectNode> getActivityTypes();
 }

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -84,9 +84,9 @@ public class MarketoServiceImpl implements MarketoService
     }
 
     @Override
-    public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
     {
-        final String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
+        final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
             marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -236,4 +236,10 @@ public class MarketoServiceImpl implements MarketoService
     {
         return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
     }
+
+    @Override
+    public Iterable<ObjectNode> getActivityTypes()
+    {
+        return marketoRestClient.getActivityTypes();
+    }
 }

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -1,45 +1,151 @@
 package org.embulk.input.marketo.delegate;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
+import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.type.Types;
 import org.joda.time.DateTime;
-import org.slf4j.Logger;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
 /**
  * Created by tai.khuu on 9/18/17.
  */
 public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
 {
-    private static final Logger LOGGER = Exec.getLogger(ActivityBulkExtractInputPlugin.class);
     public static final String INCREMENTAL_COLUMN = "activityDate";
     public static final String UID_COLUMN = "marketoGUID";
 
-    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask {}
+    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
+    {
+        @Config("activity_type_ids")
+        @ConfigDefault("[]")
+        List<String> getActivityTypeIds();
+
+        @ConfigDefault("[]")
+        List<Integer> getActTypeIds();
+
+        void setActTypeIds(List<Integer> activityIds);
+    }
 
     @Override
     public void validateInputTask(PluginTask task)
     {
         task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
         task.setUidColumn(Optional.of(UID_COLUMN));
+        if (!task.getActivityTypeIds().isEmpty()) {
+            List<Integer> activityIds = checkValidActivityTypeIds(task);
+
+            // check input with values from server
+            try (MarketoRestClient restClient = createMarketoRestClient(task)) {
+                MarketoService marketoService = new MarketoServiceImpl(restClient);
+                Iterable<ObjectNode> nodes = marketoService.getActivityTypes();
+                if (nodes != null) {
+                    checkValidActivityTypeIds(nodes, activityIds);
+                }
+                // ignorable if unable to get activity type ids. If thing gone wrong, the bulk extract will throw errors
+            }
+
+            task.setActTypeIds(activityIds);
+        }
+        else {
+            task.setActTypeIds(new ArrayList<Integer>());
+        }
         super.validateInputTask(task);
     }
 
+    /**
+     * Check if user input activity_type_ids valid
+     * @param task
+     * @return values transformed to array of Integer
+     */
+    private List<Integer> checkValidActivityTypeIds(PluginTask task)
+    {
+        List<String> invalidIds = new ArrayList<>();
+        for (String id : task.getActivityTypeIds()) {
+            if (StringUtils.isBlank(id) || !StringUtils.isNumeric(StringUtils.trimToEmpty(id))) {
+                invalidIds.add(id);
+            }
+        }
+
+        if (!invalidIds.isEmpty()) {
+            throw new ConfigException(MessageFormat.format("Invalid activity type id: [{0}]", StringUtils.join(invalidIds, ", ")));
+        }
+
+        // transform and set
+        List<Integer> activityIds = new ArrayList<>();
+        for (String id : task.getActivityTypeIds()) {
+            activityIds.add(Integer.valueOf(StringUtils.trimToEmpty(id)));
+        }
+
+        return activityIds;
+    }
+
+    @VisibleForTesting
+    protected void checkValidActivityTypeIds(Iterable<ObjectNode> nodes, List<Integer> activityIds)
+    {
+        Iterator<ObjectNode> it = nodes.iterator();
+
+        List<Integer> inputIds = new ArrayList<>(activityIds);
+
+        while (it.hasNext()) {
+            ObjectNode node = it.next();
+            int id = node.get("id").asInt(0);
+            if (id > 0) {
+                inputIds.remove(Integer.valueOf(id));
+            }
+        }
+
+        if (!inputIds.isEmpty()) {
+            throw new ConfigException(MessageFormat.format("Invalid activity type ids: [{0}], Available activity types: \n{1}",
+                    StringUtils.join(inputIds, ", "),
+                    buildActivityIdNameInfo(nodes)));
+        }
+
+    }
+
+    private String buildActivityIdNameInfo(Iterable<ObjectNode> nodes)
+    {
+        Iterator<ObjectNode> it = nodes.iterator();
+        StringBuilder messageBuilder = new StringBuilder();
+        while (it.hasNext()) {
+            ObjectNode node = it.next();
+            int id = node.get("id").asInt(0);
+            String name = node.get("name").asText("");
+            if (id > 0) {
+                messageBuilder.append("- activity id: ");
+                messageBuilder.append(String.valueOf(id));
+                messageBuilder.append(", name: ");
+                messageBuilder.append(name);
+                messageBuilder.append("\n");
+            }
+        }
+
+        return messageBuilder.toString();
+    }
+
     @Override
     protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
         try {
-            return new FileInputStream(service.extractAllActivity(fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
             throw new RuntimeException("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java

@@ -1,7 +1,5 @@
 package org.embulk.input.marketo.model;
 
-import org.embulk.input.marketo.model.filter.MarketoFilter;
-
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -16,7 +14,7 @@ public class MarketoBulkExtractRequest
 
     private Map<String, String> columnHeaderNames;
 
-    private Map<String, MarketoFilter> filter = new HashMap<>();
+    private Map<String, Object> filter = new HashMap<>();
 
     public List<String> getFields()
     {
@@ -48,12 +46,12 @@ public class MarketoBulkExtractRequest
         this.columnHeaderNames = columnHeaderNames;
     }
 
-    public Map<String, MarketoFilter> getFilter()
+    public Map<String, Object> getFilter()
     {
         return filter;
     }
 
-    public void setFilter(Map<String, MarketoFilter> filter)
+    public void setFilter(Map<String, Object> filter)
     {
         this.filter = filter;
     }

data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java

@@ -3,7 +3,7 @@ package org.embulk.input.marketo.model.filter;
 /**
  * Created by tai.khuu on 8/27/17.
  */
-public class DateRangeFilter implements MarketoFilter
+public class DateRangeFilter
 {
     private String startAt;
 

data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java

@@ -26,8 +26,8 @@ public enum MarketoRESTEndpoint
     GET_CAMPAIGN("/rest/v1/campaigns.json"),
     GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json"),
     GET_CUSTOM_OBJECT("/rest/v1/customobjects/${api_name}.json"),
-    GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json");
-
+    GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json"),
+    GET_ACTIVITY_TYPES("/rest/v1/activities/types.json");
     private String endpoint;
 
     MarketoRESTEndpoint(String endpoint)

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -22,7 +22,6 @@ import org.embulk.input.marketo.model.MarketoError;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.model.MarketoResponse;
 import org.embulk.input.marketo.model.filter.DateRangeFilter;
-import org.embulk.input.marketo.model.filter.MarketoFilter;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.type.Type;
@@ -38,6 +37,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Created by tai.khuu on 8/22/17.
@@ -187,7 +187,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
             marketoBulkExtractRequest.setFields(extractFields);
         }
         marketoBulkExtractRequest.setFormat("CSV");
-        Map<String, MarketoFilter> filterMap = new HashMap<>();
+        Map<String, Object> filterMap = new HashMap<>();
         DateRangeFilter dateRangeFilter = new DateRangeFilter();
         dateRangeFilter.setStartAt(timeFormat.format(startTime));
         dateRangeFilter.setEndAt(timeFormat.format(endTime));
@@ -196,9 +196,12 @@ public class MarketoRestClient extends MarketoBaseRestClient
         return marketoBulkExtractRequest;
     }
 
-    public String createActivityExtract(Date startTime, Date endTime)
+    public String createActivityExtract(List<Integer> activityTypeIds, Date startTime, Date endTime)
     {
         MarketoBulkExtractRequest marketoBulkExtractRequest = getMarketoBulkExtractRequest(startTime, endTime, null, "createdAt");
+        if (activityTypeIds != null && !activityTypeIds.isEmpty()) {
+            marketoBulkExtractRequest.getFilter().put("activityTypeIds", activityTypeIds);
+        }
         return sendCreateBulkExtractRequest(marketoBulkExtractRequest, MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT);
     }
 
@@ -442,7 +445,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
         // put filter params if exist.
         if (filterType != null) {
             multimap.put("filterType", filterType);
-            multimap.put("filterValues", String.join(",", filterValues));
+            multimap.put("filterValues", StringUtils.join(filterValues, ","));
         }
         return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), multimap, ObjectNode.class);
     }
@@ -534,4 +537,9 @@ public class MarketoRestClient extends MarketoBaseRestClient
     {
         return getCustomObjectRecordWithPagination(endPoint + MarketoRESTEndpoint.GET_CUSTOM_OBJECT.getEndpoint(new ImmutableMap.Builder().put("api_name", customObjectAPIName).build()), customObjectFilterType, customObjectFields, fromValue, toValue, ObjectNode.class);
     }
+
+    public Iterable<ObjectNode> getActivityTypes()
+    {
+        return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_ACTIVITY_TYPES.getEndpoint(), new ImmutableListMultimap.Builder<String, String>().put(MAX_RETURN, DEFAULT_MAX_RETURN).build(), ObjectNode.class);
+    }
 }

data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java

@@ -22,6 +22,8 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 
+import static org.mockito.ArgumentMatchers.any;
+
 /**
  * Created by tai.khuu on 10/9/17.
  */
@@ -45,12 +47,12 @@ public class MarketoServiceImplTest
     {
         Date startDate = new Date(1507223374000L);
         Date endDate = new Date(1507655374000L);
-        List<String> extractedFields = Arrays.asList("field1", "field2");
+        List<String> extractedFields = Arrays.asList("field1", "fActivityBulkExtractInputPluginTest.java:78ield2");
         String filerField = "field1";
         String exportId = "exportId";
         Mockito.when(mockMarketoRestClient.createLeadBulkExtract(Mockito.eq(startDate), Mockito.eq(endDate), Mockito.eq(extractedFields), Mockito.eq(filerField))).thenReturn(exportId);
         ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
-        Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
+        Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
         File file = marketoService.extractLead(startDate, endDate, extractedFields, filerField, 1, 3);
         Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
         Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startLeadBulkExtract(Mockito.eq(exportId));
@@ -62,11 +64,12 @@ public class MarketoServiceImplTest
     {
         Date startDate = new Date(1507223374000L);
         Date endDate = new Date(1507655374000L);
+        List<Integer> activityTypeIds = new ArrayList<>();
         String exportId = "exportId";
-        Mockito.when(mockMarketoRestClient.createActivityExtract(Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
+        Mockito.when(mockMarketoRestClient.createActivityExtract(any(List.class), Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
         ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
-        Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
-        File file = marketoService.extractAllActivity(startDate, endDate, 1, 3);
+        Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
+        File file = marketoService.extractAllActivity(activityTypeIds, startDate, endDate, 1, 3);
         Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
         Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId));
         Mockito.verify(mockMarketoRestClient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId), Mockito.eq(1), Mockito.eq(3));

data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java

@@ -1,16 +1,22 @@
 package org.embulk.input.marketo.delegate;
 
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.BulkExtractRangeHeader;
 import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
 import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.Schema;
 import org.joda.time.DateTime;
@@ -18,14 +24,17 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.function.ThrowingRunnable;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 
 import static org.mockito.ArgumentMatchers.any;
 
@@ -34,6 +43,8 @@ import static org.mockito.ArgumentMatchers.any;
  */
 public class ActivityBulkExtractInputPluginTest
 {
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
     @Rule
     public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
 
@@ -53,15 +64,49 @@ public class ActivityBulkExtractInputPluginTest
         Mockito.doReturn(mockMarketoRestclient).when(activityBulkExtractInputPlugin).createMarketoRestClient(any(ActivityBulkExtractInputPlugin.PluginTask.class));
     }
 
+    @Test
+    public void testInvalidActivityTypeIds()
+    {
+        configSource.set("activity_type_ids", Arrays.asList(" ", "abc", "123"));
+        final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
+
+        Assert.assertThrows("Invalid activity type id: [ , abc]", ConfigException.class, new ThrowingRunnable()
+        {
+            @Override
+            public void run() throws Throwable
+            {
+                activityBulkExtractInputPlugin.validateInputTask(task);
+            }
+        });
+    }
+
+    @Test
+    public void testActivityTypeIdsValid() throws IOException
+    {
+        configSource.set("activity_type_ids", Arrays.asList("1", "2", "3 "));
+        final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
+
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/activity_types.json"), javaType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+
+        Mockito.when(mockMarketoRestclient.getActivityTypes()).thenReturn(mockRecordPagingIterable);
+        activityBulkExtractInputPlugin.validateInputTask(task);
+    }
+
     @Test
     public void testRun() throws InterruptedException
     {
         ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
+
         DateTime startDate = new DateTime(task.getFromDate());
+        List<Integer> activityTypeIds = new ArrayList<>();
+
         PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
         String exportId1 = "exportId1";
         String exportId2 = "exportId2";
-        Mockito.when(mockMarketoRestclient.createActivityExtract(any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
+        Mockito.when(mockMarketoRestclient.createActivityExtract(any(List.class), any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
         Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract1.csv"));
         Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract2.csv"));
         ServiceResponseMapper<? extends ValueLocator> mapper = activityBulkExtractInputPlugin.buildServiceResponseMapper(task);
@@ -74,9 +119,9 @@ public class ActivityBulkExtractInputPluginTest
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId1), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId2));
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId2), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
-        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate.toDate(), startDate.plusDays(30).toDate());
         DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
-        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
+        Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
         ConfigDiff configDiff = activityBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
         DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
         Assert.assertEquals(df.format(startDate.plusDays(task.getFetchDays()).toDate()), configDiff.get(String.class, "from_date"));

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -35,6 +35,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
@@ -149,12 +150,13 @@ public class MarketoRestClientTest
         marketoResponse.setSuccess(true);
         Date startDate = new Date(1506865856000L);
         Date endDate = new Date(1507297856000L);
+        List<Integer> activityTypeIds = new ArrayList<>();
         ObjectNode bulkExtractResult = OBJECT_MAPPER.createObjectNode();
         bulkExtractResult.set("exportId", new TextNode("bulkExtractId"));
         marketoResponse.setResult(Arrays.asList(bulkExtractResult));
         ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
         Mockito.doReturn(marketoResponse).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), argumentCaptor.capture(), Mockito.any(MarketoResponseJetty92EntityReader.class));
-        String bulkExtractId = marketoRestClient.createActivityExtract(startDate, endDate);
+        String bulkExtractId = marketoRestClient.createActivityExtract(activityTypeIds, startDate, endDate);
         Assert.assertEquals("bulkExtractId", bulkExtractId);
         String postContent = argumentCaptor.getValue();
         ObjectNode marketoBulkExtractRequest = (ObjectNode) OBJECT_MAPPER.readTree(postContent);

data/src/test/resources/fixtures/activity_types.json

@@ -0,0 +1,22 @@
+[
+  {
+    "id": 1,
+    "name": "bb"
+  },
+  {
+    "id": 2,
+    "name": "TD Output Test aa"
+  },
+  {
+    "id": 3,
+    "name": "Bill_progream a"
+  },
+  {
+    "id": 4,
+    "name": "Bill_progream b"
+  },
+  {
+    "id": 5,
+    "name": "Bill_progream c"
+  }
+]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.15
+  version: 0.6.16
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-20 00:00:00.000000000 Z
+date: 2019-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -89,7 +89,6 @@ files:
 - src/main/java/org/embulk/input/marketo/model/MarketoField.java
 - src/main/java/org/embulk/input/marketo/model/MarketoResponse.java
 - src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java
-- src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java
 - src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java
 - src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java
 - src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java
@@ -115,6 +114,7 @@ files:
 - src/test/resources/config/rest_config.yaml
 - src/test/resources/fixtures/activity_extract1.csv
 - src/test/resources/fixtures/activity_extract2.csv
+- src/test/resources/fixtures/activity_types.json
 - src/test/resources/fixtures/all_program_full.json
 - src/test/resources/fixtures/campaign_response.json
 - src/test/resources/fixtures/campaign_response_full.json
@@ -137,9 +137,9 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.15.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.16.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo

data/src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java

@@ -1,8 +0,0 @@
-package org.embulk.input.marketo.model.filter;
-
-/**
- * Created by tai.khuu on 8/27/17.
- */
-public interface MarketoFilter
-{
-}