embulk-input-marketo 0.6.15 → 0.6.16

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a1ecb21401432881f5191869093f9e26d7eecd77
4
- data.tar.gz: 80bce389f5f836747a900ce87d2d637125df6f0b
3
+ metadata.gz: cde8656ca8356a00c7e23c93784ad972b11aafc7
4
+ data.tar.gz: f2d80312b4daed3022855c36a3101a41c107f52d
5
5
  SHA512:
6
- metadata.gz: 290ecc3ed678f765358615a0fb2e71bc61142999cf5cae9afa1c4aa60619df2e37778a884201f1505c478aef1d29311afd460f16d705785fc71a4f46f3f9c50c
7
- data.tar.gz: 03e2c3625ade8587e3c1730f8ce3a6ffb3baa77195c7a26030fe988cdd22486150862e759c752f27c91ec58cdb20fb2b76441ff46f289936b38128d4b2de202c
6
+ metadata.gz: 626137ff8225c043183e3739a4b29d2f085b722ec5569c64b172c7b9ce57798a759abef70e0f207bb4bb727a252587ca2bc199e3c21dc3f640bd0d1248607dff
7
+ data.tar.gz: a03624d1a98621cb4f32e0fcda6b2fe6a29f45879af7324f91e4dd09a3670e5fe0ab89cf5e83a0c9ee500ae58c828ff0bf9779fce9515cc813836c97cfc53303
@@ -1,3 +1,6 @@
1
+ ## 0.6.16 - 2019-12-03
2
+ * [enhancement] Added support for ActivityTypeIds filter PR [#96](https://github.com/treasure-data/embulk-input-marketo/pull/96)
3
+
1
4
  ## 0.6.15 - 2019-09-20
2
5
  * [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
3
6
 
data/README.md CHANGED
@@ -95,6 +95,8 @@ Incremental support: yes
95
95
 
96
96
  Range ingestion: yes
97
97
 
98
+ Filter by specific activity type ids: yes. See [#95](https://github.com/treasure-data/embulk-input-marketo/issues/95)
99
+
98
100
  ### Campaign
99
101
 
100
102
  Campaign extract all campaign data from Marketo
@@ -16,7 +16,7 @@ repositories {
16
16
  configurations {
17
17
  provided
18
18
  }
19
- version = "0.6.15"
19
+ version = "0.6.16"
20
20
  sourceCompatibility = 1.7
21
21
  targetCompatibility = 1.7
22
22
 
@@ -2,6 +2,7 @@ package org.embulk.input.marketo;
2
2
 
3
3
  import com.fasterxml.jackson.databind.node.ObjectNode;
4
4
  import org.embulk.input.marketo.model.MarketoField;
5
+ import org.embulk.input.marketo.model.MarketoResponse;
5
6
 
6
7
  import java.io.File;
7
8
  import java.util.Date;
@@ -16,7 +17,7 @@ public interface MarketoService
16
17
 
17
18
  File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
18
19
 
19
- File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
20
+ File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
20
21
 
21
22
  Iterable<ObjectNode> getAllListLead(List<String> extractFields);
22
23
 
@@ -33,4 +34,6 @@ public interface MarketoService
33
34
  Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
34
35
 
35
36
  List<MarketoField> describeCustomObject(String customObjectAPIName);
37
+
38
+ Iterable<ObjectNode> getActivityTypes();
36
39
  }
@@ -84,9 +84,9 @@ public class MarketoServiceImpl implements MarketoService
84
84
  }
85
85
 
86
86
  @Override
87
- public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
87
+ public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
88
88
  {
89
- final String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
89
+ final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
90
90
  marketoRestClient.startActitvityBulkExtract(exportID);
91
91
  try {
92
92
  marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -236,4 +236,10 @@ public class MarketoServiceImpl implements MarketoService
236
236
  {
237
237
  return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
238
238
  }
239
+
240
+ @Override
241
+ public Iterable<ObjectNode> getActivityTypes()
242
+ {
243
+ return marketoRestClient.getActivityTypes();
244
+ }
239
245
  }
@@ -1,45 +1,151 @@
1
1
  package org.embulk.input.marketo.delegate;
2
2
 
3
+ import com.fasterxml.jackson.databind.node.ObjectNode;
4
+ import com.google.common.annotations.VisibleForTesting;
3
5
  import com.google.common.base.Optional;
6
+ import org.apache.commons.lang3.StringUtils;
4
7
  import org.embulk.base.restclient.ServiceResponseMapper;
5
8
  import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
6
9
  import org.embulk.base.restclient.record.ValueLocator;
10
+ import org.embulk.config.Config;
11
+ import org.embulk.config.ConfigDefault;
12
+ import org.embulk.config.ConfigException;
7
13
  import org.embulk.input.marketo.MarketoService;
14
+ import org.embulk.input.marketo.MarketoServiceImpl;
8
15
  import org.embulk.input.marketo.MarketoUtils;
9
- import org.embulk.spi.DataException;
10
- import org.embulk.spi.Exec;
16
+ import org.embulk.input.marketo.rest.MarketoRestClient;
11
17
  import org.embulk.spi.type.Types;
12
18
  import org.joda.time.DateTime;
13
- import org.slf4j.Logger;
14
19
 
15
20
  import java.io.FileInputStream;
16
21
  import java.io.FileNotFoundException;
17
22
  import java.io.InputStream;
23
+ import java.text.MessageFormat;
24
+ import java.util.ArrayList;
25
+ import java.util.Iterator;
26
+ import java.util.List;
18
27
 
19
28
  /**
20
29
  * Created by tai.khuu on 9/18/17.
21
30
  */
22
31
  public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
23
32
  {
24
- private static final Logger LOGGER = Exec.getLogger(ActivityBulkExtractInputPlugin.class);
25
33
  public static final String INCREMENTAL_COLUMN = "activityDate";
26
34
  public static final String UID_COLUMN = "marketoGUID";
27
35
 
28
- public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask {}
36
+ public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
37
+ {
38
+ @Config("activity_type_ids")
39
+ @ConfigDefault("[]")
40
+ List<String> getActivityTypeIds();
41
+
42
+ @ConfigDefault("[]")
43
+ List<Integer> getActTypeIds();
44
+
45
+ void setActTypeIds(List<Integer> activityIds);
46
+ }
29
47
 
30
48
  @Override
31
49
  public void validateInputTask(PluginTask task)
32
50
  {
33
51
  task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
34
52
  task.setUidColumn(Optional.of(UID_COLUMN));
53
+ if (!task.getActivityTypeIds().isEmpty()) {
54
+ List<Integer> activityIds = checkValidActivityTypeIds(task);
55
+
56
+ // check input with values from server
57
+ try (MarketoRestClient restClient = createMarketoRestClient(task)) {
58
+ MarketoService marketoService = new MarketoServiceImpl(restClient);
59
+ Iterable<ObjectNode> nodes = marketoService.getActivityTypes();
60
+ if (nodes != null) {
61
+ checkValidActivityTypeIds(nodes, activityIds);
62
+ }
63
+ // ignorable if unable to get activity type ids. If thing gone wrong, the bulk extract will throw errors
64
+ }
65
+
66
+ task.setActTypeIds(activityIds);
67
+ }
68
+ else {
69
+ task.setActTypeIds(new ArrayList<Integer>());
70
+ }
35
71
  super.validateInputTask(task);
36
72
  }
37
73
 
74
+ /**
75
+ * Check if user input activity_type_ids valid
76
+ * @param task
77
+ * @return values transformed to array of Integer
78
+ */
79
+ private List<Integer> checkValidActivityTypeIds(PluginTask task)
80
+ {
81
+ List<String> invalidIds = new ArrayList<>();
82
+ for (String id : task.getActivityTypeIds()) {
83
+ if (StringUtils.isBlank(id) || !StringUtils.isNumeric(StringUtils.trimToEmpty(id))) {
84
+ invalidIds.add(id);
85
+ }
86
+ }
87
+
88
+ if (!invalidIds.isEmpty()) {
89
+ throw new ConfigException(MessageFormat.format("Invalid activity type id: [{0}]", StringUtils.join(invalidIds, ", ")));
90
+ }
91
+
92
+ // transform and set
93
+ List<Integer> activityIds = new ArrayList<>();
94
+ for (String id : task.getActivityTypeIds()) {
95
+ activityIds.add(Integer.valueOf(StringUtils.trimToEmpty(id)));
96
+ }
97
+
98
+ return activityIds;
99
+ }
100
+
101
+ @VisibleForTesting
102
+ protected void checkValidActivityTypeIds(Iterable<ObjectNode> nodes, List<Integer> activityIds)
103
+ {
104
+ Iterator<ObjectNode> it = nodes.iterator();
105
+
106
+ List<Integer> inputIds = new ArrayList<>(activityIds);
107
+
108
+ while (it.hasNext()) {
109
+ ObjectNode node = it.next();
110
+ int id = node.get("id").asInt(0);
111
+ if (id > 0) {
112
+ inputIds.remove(Integer.valueOf(id));
113
+ }
114
+ }
115
+
116
+ if (!inputIds.isEmpty()) {
117
+ throw new ConfigException(MessageFormat.format("Invalid activity type ids: [{0}], Available activity types: \n{1}",
118
+ StringUtils.join(inputIds, ", "),
119
+ buildActivityIdNameInfo(nodes)));
120
+ }
121
+
122
+ }
123
+
124
+ private String buildActivityIdNameInfo(Iterable<ObjectNode> nodes)
125
+ {
126
+ Iterator<ObjectNode> it = nodes.iterator();
127
+ StringBuilder messageBuilder = new StringBuilder();
128
+ while (it.hasNext()) {
129
+ ObjectNode node = it.next();
130
+ int id = node.get("id").asInt(0);
131
+ String name = node.get("name").asText("");
132
+ if (id > 0) {
133
+ messageBuilder.append("- activity id: ");
134
+ messageBuilder.append(String.valueOf(id));
135
+ messageBuilder.append(", name: ");
136
+ messageBuilder.append(name);
137
+ messageBuilder.append("\n");
138
+ }
139
+ }
140
+
141
+ return messageBuilder.toString();
142
+ }
143
+
38
144
  @Override
39
145
  protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
40
146
  {
41
147
  try {
42
- return new FileInputStream(service.extractAllActivity(fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
148
+ return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
43
149
  }
44
150
  catch (FileNotFoundException e) {
45
151
  throw new RuntimeException("Exception when trying to extract activity", e);
@@ -1,7 +1,5 @@
1
1
  package org.embulk.input.marketo.model;
2
2
 
3
- import org.embulk.input.marketo.model.filter.MarketoFilter;
4
-
5
3
  import java.util.HashMap;
6
4
  import java.util.List;
7
5
  import java.util.Map;
@@ -16,7 +14,7 @@ public class MarketoBulkExtractRequest
16
14
 
17
15
  private Map<String, String> columnHeaderNames;
18
16
 
19
- private Map<String, MarketoFilter> filter = new HashMap<>();
17
+ private Map<String, Object> filter = new HashMap<>();
20
18
 
21
19
  public List<String> getFields()
22
20
  {
@@ -48,12 +46,12 @@ public class MarketoBulkExtractRequest
48
46
  this.columnHeaderNames = columnHeaderNames;
49
47
  }
50
48
 
51
- public Map<String, MarketoFilter> getFilter()
49
+ public Map<String, Object> getFilter()
52
50
  {
53
51
  return filter;
54
52
  }
55
53
 
56
- public void setFilter(Map<String, MarketoFilter> filter)
54
+ public void setFilter(Map<String, Object> filter)
57
55
  {
58
56
  this.filter = filter;
59
57
  }
@@ -3,7 +3,7 @@ package org.embulk.input.marketo.model.filter;
3
3
  /**
4
4
  * Created by tai.khuu on 8/27/17.
5
5
  */
6
- public class DateRangeFilter implements MarketoFilter
6
+ public class DateRangeFilter
7
7
  {
8
8
  private String startAt;
9
9
 
@@ -26,8 +26,8 @@ public enum MarketoRESTEndpoint
26
26
  GET_CAMPAIGN("/rest/v1/campaigns.json"),
27
27
  GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json"),
28
28
  GET_CUSTOM_OBJECT("/rest/v1/customobjects/${api_name}.json"),
29
- GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json");
30
-
29
+ GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json"),
30
+ GET_ACTIVITY_TYPES("/rest/v1/activities/types.json");
31
31
  private String endpoint;
32
32
 
33
33
  MarketoRESTEndpoint(String endpoint)
@@ -22,7 +22,6 @@ import org.embulk.input.marketo.model.MarketoError;
22
22
  import org.embulk.input.marketo.model.MarketoField;
23
23
  import org.embulk.input.marketo.model.MarketoResponse;
24
24
  import org.embulk.input.marketo.model.filter.DateRangeFilter;
25
- import org.embulk.input.marketo.model.filter.MarketoFilter;
26
25
  import org.embulk.spi.DataException;
27
26
  import org.embulk.spi.Exec;
28
27
  import org.embulk.spi.type.Type;
@@ -38,6 +37,7 @@ import java.util.Date;
38
37
  import java.util.HashMap;
39
38
  import java.util.List;
40
39
  import java.util.Map;
40
+ import java.util.stream.Collectors;
41
41
 
42
42
  /**
43
43
  * Created by tai.khuu on 8/22/17.
@@ -187,7 +187,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
187
187
  marketoBulkExtractRequest.setFields(extractFields);
188
188
  }
189
189
  marketoBulkExtractRequest.setFormat("CSV");
190
- Map<String, MarketoFilter> filterMap = new HashMap<>();
190
+ Map<String, Object> filterMap = new HashMap<>();
191
191
  DateRangeFilter dateRangeFilter = new DateRangeFilter();
192
192
  dateRangeFilter.setStartAt(timeFormat.format(startTime));
193
193
  dateRangeFilter.setEndAt(timeFormat.format(endTime));
@@ -196,9 +196,12 @@ public class MarketoRestClient extends MarketoBaseRestClient
196
196
  return marketoBulkExtractRequest;
197
197
  }
198
198
 
199
- public String createActivityExtract(Date startTime, Date endTime)
199
+ public String createActivityExtract(List<Integer> activityTypeIds, Date startTime, Date endTime)
200
200
  {
201
201
  MarketoBulkExtractRequest marketoBulkExtractRequest = getMarketoBulkExtractRequest(startTime, endTime, null, "createdAt");
202
+ if (activityTypeIds != null && !activityTypeIds.isEmpty()) {
203
+ marketoBulkExtractRequest.getFilter().put("activityTypeIds", activityTypeIds);
204
+ }
202
205
  return sendCreateBulkExtractRequest(marketoBulkExtractRequest, MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT);
203
206
  }
204
207
 
@@ -442,7 +445,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
442
445
  // put filter params if exist.
443
446
  if (filterType != null) {
444
447
  multimap.put("filterType", filterType);
445
- multimap.put("filterValues", String.join(",", filterValues));
448
+ multimap.put("filterValues", StringUtils.join(filterValues, ","));
446
449
  }
447
450
  return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), multimap, ObjectNode.class);
448
451
  }
@@ -534,4 +537,9 @@ public class MarketoRestClient extends MarketoBaseRestClient
534
537
  {
535
538
  return getCustomObjectRecordWithPagination(endPoint + MarketoRESTEndpoint.GET_CUSTOM_OBJECT.getEndpoint(new ImmutableMap.Builder().put("api_name", customObjectAPIName).build()), customObjectFilterType, customObjectFields, fromValue, toValue, ObjectNode.class);
536
539
  }
540
+
541
+ public Iterable<ObjectNode> getActivityTypes()
542
+ {
543
+ return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_ACTIVITY_TYPES.getEndpoint(), new ImmutableListMultimap.Builder<String, String>().put(MAX_RETURN, DEFAULT_MAX_RETURN).build(), ObjectNode.class);
544
+ }
537
545
  }
@@ -22,6 +22,8 @@ import java.util.Date;
22
22
  import java.util.Iterator;
23
23
  import java.util.List;
24
24
 
25
+ import static org.mockito.ArgumentMatchers.any;
26
+
25
27
  /**
26
28
  * Created by tai.khuu on 10/9/17.
27
29
  */
@@ -45,12 +47,12 @@ public class MarketoServiceImplTest
45
47
  {
46
48
  Date startDate = new Date(1507223374000L);
47
49
  Date endDate = new Date(1507655374000L);
48
- List<String> extractedFields = Arrays.asList("field1", "field2");
50
+ List<String> extractedFields = Arrays.asList("field1", "fActivityBulkExtractInputPluginTest.java:78ield2");
49
51
  String filerField = "field1";
50
52
  String exportId = "exportId";
51
53
  Mockito.when(mockMarketoRestClient.createLeadBulkExtract(Mockito.eq(startDate), Mockito.eq(endDate), Mockito.eq(extractedFields), Mockito.eq(filerField))).thenReturn(exportId);
52
54
  ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
53
- Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
55
+ Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
54
56
  File file = marketoService.extractLead(startDate, endDate, extractedFields, filerField, 1, 3);
55
57
  Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
56
58
  Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startLeadBulkExtract(Mockito.eq(exportId));
@@ -62,11 +64,12 @@ public class MarketoServiceImplTest
62
64
  {
63
65
  Date startDate = new Date(1507223374000L);
64
66
  Date endDate = new Date(1507655374000L);
67
+ List<Integer> activityTypeIds = new ArrayList<>();
65
68
  String exportId = "exportId";
66
- Mockito.when(mockMarketoRestClient.createActivityExtract(Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
69
+ Mockito.when(mockMarketoRestClient.createActivityExtract(any(List.class), Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
67
70
  ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
68
- Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), Mockito.any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
69
- File file = marketoService.extractAllActivity(startDate, endDate, 1, 3);
71
+ Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
72
+ File file = marketoService.extractAllActivity(activityTypeIds, startDate, endDate, 1, 3);
70
73
  Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
71
74
  Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId));
72
75
  Mockito.verify(mockMarketoRestClient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId), Mockito.eq(1), Mockito.eq(3));
@@ -1,16 +1,22 @@
1
1
  package org.embulk.input.marketo.delegate;
2
2
 
3
+ import com.fasterxml.jackson.databind.JavaType;
4
+ import com.fasterxml.jackson.databind.ObjectMapper;
5
+ import com.fasterxml.jackson.databind.node.ObjectNode;
3
6
  import org.embulk.EmbulkTestRuntime;
4
7
  import org.embulk.base.restclient.ServiceResponseMapper;
5
8
  import org.embulk.base.restclient.record.ValueLocator;
6
9
  import org.embulk.config.ConfigDiff;
10
+ import org.embulk.config.ConfigException;
7
11
  import org.embulk.config.ConfigLoader;
8
12
  import org.embulk.config.ConfigSource;
9
13
  import org.embulk.config.TaskReport;
10
14
  import org.embulk.input.marketo.MarketoUtils;
11
15
  import org.embulk.input.marketo.model.BulkExtractRangeHeader;
12
16
  import org.embulk.input.marketo.rest.MarketoRestClient;
17
+ import org.embulk.input.marketo.rest.RecordPagingIterable;
13
18
  import org.embulk.spi.Column;
19
+ import org.embulk.spi.Exec;
14
20
  import org.embulk.spi.PageBuilder;
15
21
  import org.embulk.spi.Schema;
16
22
  import org.joda.time.DateTime;
@@ -18,14 +24,17 @@ import org.junit.Assert;
18
24
  import org.junit.Before;
19
25
  import org.junit.Rule;
20
26
  import org.junit.Test;
27
+ import org.junit.function.ThrowingRunnable;
21
28
  import org.mockito.ArgumentCaptor;
22
29
  import org.mockito.Mockito;
23
30
 
24
31
  import java.io.IOException;
25
32
  import java.text.DateFormat;
26
33
  import java.text.SimpleDateFormat;
34
+ import java.util.ArrayList;
27
35
  import java.util.Arrays;
28
36
  import java.util.Date;
37
+ import java.util.List;
29
38
 
30
39
  import static org.mockito.ArgumentMatchers.any;
31
40
 
@@ -34,6 +43,8 @@ import static org.mockito.ArgumentMatchers.any;
34
43
  */
35
44
  public class ActivityBulkExtractInputPluginTest
36
45
  {
46
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
47
+
37
48
  @Rule
38
49
  public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
39
50
 
@@ -53,15 +64,49 @@ public class ActivityBulkExtractInputPluginTest
53
64
  Mockito.doReturn(mockMarketoRestclient).when(activityBulkExtractInputPlugin).createMarketoRestClient(any(ActivityBulkExtractInputPlugin.PluginTask.class));
54
65
  }
55
66
 
67
+ @Test
68
+ public void testInvalidActivityTypeIds()
69
+ {
70
+ configSource.set("activity_type_ids", Arrays.asList(" ", "abc", "123"));
71
+ final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
72
+
73
+ Assert.assertThrows("Invalid activity type id: [ , abc]", ConfigException.class, new ThrowingRunnable()
74
+ {
75
+ @Override
76
+ public void run() throws Throwable
77
+ {
78
+ activityBulkExtractInputPlugin.validateInputTask(task);
79
+ }
80
+ });
81
+ }
82
+
83
+ @Test
84
+ public void testActivityTypeIdsValid() throws IOException
85
+ {
86
+ configSource.set("activity_type_ids", Arrays.asList("1", "2", "3 "));
87
+ final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
88
+
89
+ RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
90
+ JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
91
+ List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/activity_types.json"), javaType);
92
+ Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
93
+
94
+ Mockito.when(mockMarketoRestclient.getActivityTypes()).thenReturn(mockRecordPagingIterable);
95
+ activityBulkExtractInputPlugin.validateInputTask(task);
96
+ }
97
+
56
98
  @Test
57
99
  public void testRun() throws InterruptedException
58
100
  {
59
101
  ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
102
+
60
103
  DateTime startDate = new DateTime(task.getFromDate());
104
+ List<Integer> activityTypeIds = new ArrayList<>();
105
+
61
106
  PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
62
107
  String exportId1 = "exportId1";
63
108
  String exportId2 = "exportId2";
64
- Mockito.when(mockMarketoRestclient.createActivityExtract(any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
109
+ Mockito.when(mockMarketoRestclient.createActivityExtract(any(List.class), any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
65
110
  Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract1.csv"));
66
111
  Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract2.csv"));
67
112
  ServiceResponseMapper<? extends ValueLocator> mapper = activityBulkExtractInputPlugin.buildServiceResponseMapper(task);
@@ -74,9 +119,9 @@ public class ActivityBulkExtractInputPluginTest
74
119
  Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId1), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
75
120
  Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId2));
76
121
  Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId2), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
77
- Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
122
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate.toDate(), startDate.plusDays(30).toDate());
78
123
  DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
79
- Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
124
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
80
125
  ConfigDiff configDiff = activityBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
81
126
  DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
82
127
  Assert.assertEquals(df.format(startDate.plusDays(task.getFetchDays()).toDate()), configDiff.get(String.class, "from_date"));
@@ -35,6 +35,7 @@ import java.io.IOException;
35
35
  import java.io.InputStream;
36
36
  import java.nio.ByteBuffer;
37
37
  import java.text.SimpleDateFormat;
38
+ import java.util.ArrayList;
38
39
  import java.util.Arrays;
39
40
  import java.util.Date;
40
41
  import java.util.HashMap;
@@ -149,12 +150,13 @@ public class MarketoRestClientTest
149
150
  marketoResponse.setSuccess(true);
150
151
  Date startDate = new Date(1506865856000L);
151
152
  Date endDate = new Date(1507297856000L);
153
+ List<Integer> activityTypeIds = new ArrayList<>();
152
154
  ObjectNode bulkExtractResult = OBJECT_MAPPER.createObjectNode();
153
155
  bulkExtractResult.set("exportId", new TextNode("bulkExtractId"));
154
156
  marketoResponse.setResult(Arrays.asList(bulkExtractResult));
155
157
  ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
156
158
  Mockito.doReturn(marketoResponse).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), argumentCaptor.capture(), Mockito.any(MarketoResponseJetty92EntityReader.class));
157
- String bulkExtractId = marketoRestClient.createActivityExtract(startDate, endDate);
159
+ String bulkExtractId = marketoRestClient.createActivityExtract(activityTypeIds, startDate, endDate);
158
160
  Assert.assertEquals("bulkExtractId", bulkExtractId);
159
161
  String postContent = argumentCaptor.getValue();
160
162
  ObjectNode marketoBulkExtractRequest = (ObjectNode) OBJECT_MAPPER.readTree(postContent);
@@ -0,0 +1,22 @@
1
+ [
2
+ {
3
+ "id": 1,
4
+ "name": "bb"
5
+ },
6
+ {
7
+ "id": 2,
8
+ "name": "TD Output Test aa"
9
+ },
10
+ {
11
+ "id": 3,
12
+ "name": "Bill_progream a"
13
+ },
14
+ {
15
+ "id": 4,
16
+ "name": "Bill_progream b"
17
+ },
18
+ {
19
+ "id": 5,
20
+ "name": "Bill_progream c"
21
+ }
22
+ ]
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-marketo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.15
4
+ version: 0.6.16
5
5
  platform: ruby
6
6
  authors:
7
7
  - uu59
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-09-20 00:00:00.000000000 Z
13
+ date: 2019-12-03 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  requirement: !ruby/object:Gem::Requirement
@@ -89,7 +89,6 @@ files:
89
89
  - src/main/java/org/embulk/input/marketo/model/MarketoField.java
90
90
  - src/main/java/org/embulk/input/marketo/model/MarketoResponse.java
91
91
  - src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java
92
- - src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java
93
92
  - src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java
94
93
  - src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java
95
94
  - src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java
@@ -115,6 +114,7 @@ files:
115
114
  - src/test/resources/config/rest_config.yaml
116
115
  - src/test/resources/fixtures/activity_extract1.csv
117
116
  - src/test/resources/fixtures/activity_extract2.csv
117
+ - src/test/resources/fixtures/activity_types.json
118
118
  - src/test/resources/fixtures/all_program_full.json
119
119
  - src/test/resources/fixtures/campaign_response.json
120
120
  - src/test/resources/fixtures/campaign_response_full.json
@@ -137,9 +137,9 @@ files:
137
137
  - src/test/resources/fixtures/program_response.json
138
138
  - classpath/jetty-io-9.2.14.v20151106.jar
139
139
  - classpath/jetty-util-9.2.14.v20151106.jar
140
- - classpath/embulk-input-marketo-0.6.15.jar
141
140
  - classpath/jetty-http-9.2.14.v20151106.jar
142
141
  - classpath/jetty-client-9.2.14.v20151106.jar
142
+ - classpath/embulk-input-marketo-0.6.16.jar
143
143
  - classpath/embulk-base-restclient-0.5.3.jar
144
144
  - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
145
145
  homepage: https://github.com/treasure-data/embulk-input-marketo
@@ -1,8 +0,0 @@
1
- package org.embulk.input.marketo.model.filter;
2
-
3
- /**
4
- * Created by tai.khuu on 8/27/17.
5
- */
6
- public interface MarketoFilter
7
- {
8
- }