embulk-input-marketo 0.6.15 → 0.6.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +2 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/marketo/MarketoService.java +4 -1
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +8 -2
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +112 -6
- data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +3 -5
- data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +1 -1
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +2 -2
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +12 -4
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +8 -5
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +48 -3
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +3 -1
- data/src/test/resources/fixtures/activity_types.json +22 -0
- metadata +4 -4
- data/src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cde8656ca8356a00c7e23c93784ad972b11aafc7
|
4
|
+
data.tar.gz: f2d80312b4daed3022855c36a3101a41c107f52d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 626137ff8225c043183e3739a4b29d2f085b722ec5569c64b172c7b9ce57798a759abef70e0f207bb4bb727a252587ca2bc199e3c21dc3f640bd0d1248607dff
|
7
|
+
data.tar.gz: a03624d1a98621cb4f32e0fcda6b2fe6a29f45879af7324f91e4dd09a3670e5fe0ab89cf5e83a0c9ee500ae58c828ff0bf9779fce9515cc813836c97cfc53303
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.6.16 - 2019-12-03
|
2
|
+
* [enhancement] Added support for ActivityTypeIds filter PR [#96](https://github.com/treasure-data/embulk-input-marketo/pull/96)
|
3
|
+
|
1
4
|
## 0.6.15 - 2019-09-20
|
2
5
|
* [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
|
3
6
|
|
data/README.md
CHANGED
data/build.gradle
CHANGED
@@ -2,6 +2,7 @@ package org.embulk.input.marketo;
|
|
2
2
|
|
3
3
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
4
|
import org.embulk.input.marketo.model.MarketoField;
|
5
|
+
import org.embulk.input.marketo.model.MarketoResponse;
|
5
6
|
|
6
7
|
import java.io.File;
|
7
8
|
import java.util.Date;
|
@@ -16,7 +17,7 @@ public interface MarketoService
|
|
16
17
|
|
17
18
|
File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
|
18
19
|
|
19
|
-
File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
|
20
|
+
File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
|
20
21
|
|
21
22
|
Iterable<ObjectNode> getAllListLead(List<String> extractFields);
|
22
23
|
|
@@ -33,4 +34,6 @@ public interface MarketoService
|
|
33
34
|
Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
|
34
35
|
|
35
36
|
List<MarketoField> describeCustomObject(String customObjectAPIName);
|
37
|
+
|
38
|
+
Iterable<ObjectNode> getActivityTypes();
|
36
39
|
}
|
@@ -84,9 +84,9 @@ public class MarketoServiceImpl implements MarketoService
|
|
84
84
|
}
|
85
85
|
|
86
86
|
@Override
|
87
|
-
public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
|
87
|
+
public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
|
88
88
|
{
|
89
|
-
final String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
|
89
|
+
final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
|
90
90
|
marketoRestClient.startActitvityBulkExtract(exportID);
|
91
91
|
try {
|
92
92
|
marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
|
@@ -236,4 +236,10 @@ public class MarketoServiceImpl implements MarketoService
|
|
236
236
|
{
|
237
237
|
return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
|
238
238
|
}
|
239
|
+
|
240
|
+
@Override
|
241
|
+
public Iterable<ObjectNode> getActivityTypes()
|
242
|
+
{
|
243
|
+
return marketoRestClient.getActivityTypes();
|
244
|
+
}
|
239
245
|
}
|
@@ -1,45 +1,151 @@
|
|
1
1
|
package org.embulk.input.marketo.delegate;
|
2
2
|
|
3
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
|
+
import com.google.common.annotations.VisibleForTesting;
|
3
5
|
import com.google.common.base.Optional;
|
6
|
+
import org.apache.commons.lang3.StringUtils;
|
4
7
|
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
8
|
import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
|
6
9
|
import org.embulk.base.restclient.record.ValueLocator;
|
10
|
+
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
12
|
+
import org.embulk.config.ConfigException;
|
7
13
|
import org.embulk.input.marketo.MarketoService;
|
14
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
8
15
|
import org.embulk.input.marketo.MarketoUtils;
|
9
|
-
import org.embulk.
|
10
|
-
import org.embulk.spi.Exec;
|
16
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
11
17
|
import org.embulk.spi.type.Types;
|
12
18
|
import org.joda.time.DateTime;
|
13
|
-
import org.slf4j.Logger;
|
14
19
|
|
15
20
|
import java.io.FileInputStream;
|
16
21
|
import java.io.FileNotFoundException;
|
17
22
|
import java.io.InputStream;
|
23
|
+
import java.text.MessageFormat;
|
24
|
+
import java.util.ArrayList;
|
25
|
+
import java.util.Iterator;
|
26
|
+
import java.util.List;
|
18
27
|
|
19
28
|
/**
|
20
29
|
* Created by tai.khuu on 9/18/17.
|
21
30
|
*/
|
22
31
|
public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
|
23
32
|
{
|
24
|
-
private static final Logger LOGGER = Exec.getLogger(ActivityBulkExtractInputPlugin.class);
|
25
33
|
public static final String INCREMENTAL_COLUMN = "activityDate";
|
26
34
|
public static final String UID_COLUMN = "marketoGUID";
|
27
35
|
|
28
|
-
public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
|
36
|
+
public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
|
37
|
+
{
|
38
|
+
@Config("activity_type_ids")
|
39
|
+
@ConfigDefault("[]")
|
40
|
+
List<String> getActivityTypeIds();
|
41
|
+
|
42
|
+
@ConfigDefault("[]")
|
43
|
+
List<Integer> getActTypeIds();
|
44
|
+
|
45
|
+
void setActTypeIds(List<Integer> activityIds);
|
46
|
+
}
|
29
47
|
|
30
48
|
@Override
|
31
49
|
public void validateInputTask(PluginTask task)
|
32
50
|
{
|
33
51
|
task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
|
34
52
|
task.setUidColumn(Optional.of(UID_COLUMN));
|
53
|
+
if (!task.getActivityTypeIds().isEmpty()) {
|
54
|
+
List<Integer> activityIds = checkValidActivityTypeIds(task);
|
55
|
+
|
56
|
+
// check input with values from server
|
57
|
+
try (MarketoRestClient restClient = createMarketoRestClient(task)) {
|
58
|
+
MarketoService marketoService = new MarketoServiceImpl(restClient);
|
59
|
+
Iterable<ObjectNode> nodes = marketoService.getActivityTypes();
|
60
|
+
if (nodes != null) {
|
61
|
+
checkValidActivityTypeIds(nodes, activityIds);
|
62
|
+
}
|
63
|
+
// ignorable if unable to get activity type ids. If thing gone wrong, the bulk extract will throw errors
|
64
|
+
}
|
65
|
+
|
66
|
+
task.setActTypeIds(activityIds);
|
67
|
+
}
|
68
|
+
else {
|
69
|
+
task.setActTypeIds(new ArrayList<Integer>());
|
70
|
+
}
|
35
71
|
super.validateInputTask(task);
|
36
72
|
}
|
37
73
|
|
74
|
+
/**
|
75
|
+
* Check if user input activity_type_ids valid
|
76
|
+
* @param task
|
77
|
+
* @return values transformed to array of Integer
|
78
|
+
*/
|
79
|
+
private List<Integer> checkValidActivityTypeIds(PluginTask task)
|
80
|
+
{
|
81
|
+
List<String> invalidIds = new ArrayList<>();
|
82
|
+
for (String id : task.getActivityTypeIds()) {
|
83
|
+
if (StringUtils.isBlank(id) || !StringUtils.isNumeric(StringUtils.trimToEmpty(id))) {
|
84
|
+
invalidIds.add(id);
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
if (!invalidIds.isEmpty()) {
|
89
|
+
throw new ConfigException(MessageFormat.format("Invalid activity type id: [{0}]", StringUtils.join(invalidIds, ", ")));
|
90
|
+
}
|
91
|
+
|
92
|
+
// transform and set
|
93
|
+
List<Integer> activityIds = new ArrayList<>();
|
94
|
+
for (String id : task.getActivityTypeIds()) {
|
95
|
+
activityIds.add(Integer.valueOf(StringUtils.trimToEmpty(id)));
|
96
|
+
}
|
97
|
+
|
98
|
+
return activityIds;
|
99
|
+
}
|
100
|
+
|
101
|
+
@VisibleForTesting
|
102
|
+
protected void checkValidActivityTypeIds(Iterable<ObjectNode> nodes, List<Integer> activityIds)
|
103
|
+
{
|
104
|
+
Iterator<ObjectNode> it = nodes.iterator();
|
105
|
+
|
106
|
+
List<Integer> inputIds = new ArrayList<>(activityIds);
|
107
|
+
|
108
|
+
while (it.hasNext()) {
|
109
|
+
ObjectNode node = it.next();
|
110
|
+
int id = node.get("id").asInt(0);
|
111
|
+
if (id > 0) {
|
112
|
+
inputIds.remove(Integer.valueOf(id));
|
113
|
+
}
|
114
|
+
}
|
115
|
+
|
116
|
+
if (!inputIds.isEmpty()) {
|
117
|
+
throw new ConfigException(MessageFormat.format("Invalid activity type ids: [{0}], Available activity types: \n{1}",
|
118
|
+
StringUtils.join(inputIds, ", "),
|
119
|
+
buildActivityIdNameInfo(nodes)));
|
120
|
+
}
|
121
|
+
|
122
|
+
}
|
123
|
+
|
124
|
+
private String buildActivityIdNameInfo(Iterable<ObjectNode> nodes)
|
125
|
+
{
|
126
|
+
Iterator<ObjectNode> it = nodes.iterator();
|
127
|
+
StringBuilder messageBuilder = new StringBuilder();
|
128
|
+
while (it.hasNext()) {
|
129
|
+
ObjectNode node = it.next();
|
130
|
+
int id = node.get("id").asInt(0);
|
131
|
+
String name = node.get("name").asText("");
|
132
|
+
if (id > 0) {
|
133
|
+
messageBuilder.append("- activity id: ");
|
134
|
+
messageBuilder.append(String.valueOf(id));
|
135
|
+
messageBuilder.append(", name: ");
|
136
|
+
messageBuilder.append(name);
|
137
|
+
messageBuilder.append("\n");
|
138
|
+
}
|
139
|
+
}
|
140
|
+
|
141
|
+
return messageBuilder.toString();
|
142
|
+
}
|
143
|
+
|
38
144
|
@Override
|
39
145
|
protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
|
40
146
|
{
|
41
147
|
try {
|
42
|
-
return new FileInputStream(service.extractAllActivity(fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
|
148
|
+
return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
|
43
149
|
}
|
44
150
|
catch (FileNotFoundException e) {
|
45
151
|
throw new RuntimeException("Exception when trying to extract activity", e);
|
@@ -1,7 +1,5 @@
|
|
1
1
|
package org.embulk.input.marketo.model;
|
2
2
|
|
3
|
-
import org.embulk.input.marketo.model.filter.MarketoFilter;
|
4
|
-
|
5
3
|
import java.util.HashMap;
|
6
4
|
import java.util.List;
|
7
5
|
import java.util.Map;
|
@@ -16,7 +14,7 @@ public class MarketoBulkExtractRequest
|
|
16
14
|
|
17
15
|
private Map<String, String> columnHeaderNames;
|
18
16
|
|
19
|
-
private Map<String,
|
17
|
+
private Map<String, Object> filter = new HashMap<>();
|
20
18
|
|
21
19
|
public List<String> getFields()
|
22
20
|
{
|
@@ -48,12 +46,12 @@ public class MarketoBulkExtractRequest
|
|
48
46
|
this.columnHeaderNames = columnHeaderNames;
|
49
47
|
}
|
50
48
|
|
51
|
-
public Map<String,
|
49
|
+
public Map<String, Object> getFilter()
|
52
50
|
{
|
53
51
|
return filter;
|
54
52
|
}
|
55
53
|
|
56
|
-
public void setFilter(Map<String,
|
54
|
+
public void setFilter(Map<String, Object> filter)
|
57
55
|
{
|
58
56
|
this.filter = filter;
|
59
57
|
}
|
@@ -26,8 +26,8 @@ public enum MarketoRESTEndpoint
|
|
26
26
|
GET_CAMPAIGN("/rest/v1/campaigns.json"),
|
27
27
|
GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json"),
|
28
28
|
GET_CUSTOM_OBJECT("/rest/v1/customobjects/${api_name}.json"),
|
29
|
-
GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json")
|
30
|
-
|
29
|
+
GET_CUSTOM_OBJECT_DESCRIBE("/rest/v1/customobjects/${api_name}/describe.json"),
|
30
|
+
GET_ACTIVITY_TYPES("/rest/v1/activities/types.json");
|
31
31
|
private String endpoint;
|
32
32
|
|
33
33
|
MarketoRESTEndpoint(String endpoint)
|
@@ -22,7 +22,6 @@ import org.embulk.input.marketo.model.MarketoError;
|
|
22
22
|
import org.embulk.input.marketo.model.MarketoField;
|
23
23
|
import org.embulk.input.marketo.model.MarketoResponse;
|
24
24
|
import org.embulk.input.marketo.model.filter.DateRangeFilter;
|
25
|
-
import org.embulk.input.marketo.model.filter.MarketoFilter;
|
26
25
|
import org.embulk.spi.DataException;
|
27
26
|
import org.embulk.spi.Exec;
|
28
27
|
import org.embulk.spi.type.Type;
|
@@ -38,6 +37,7 @@ import java.util.Date;
|
|
38
37
|
import java.util.HashMap;
|
39
38
|
import java.util.List;
|
40
39
|
import java.util.Map;
|
40
|
+
import java.util.stream.Collectors;
|
41
41
|
|
42
42
|
/**
|
43
43
|
* Created by tai.khuu on 8/22/17.
|
@@ -187,7 +187,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
187
187
|
marketoBulkExtractRequest.setFields(extractFields);
|
188
188
|
}
|
189
189
|
marketoBulkExtractRequest.setFormat("CSV");
|
190
|
-
Map<String,
|
190
|
+
Map<String, Object> filterMap = new HashMap<>();
|
191
191
|
DateRangeFilter dateRangeFilter = new DateRangeFilter();
|
192
192
|
dateRangeFilter.setStartAt(timeFormat.format(startTime));
|
193
193
|
dateRangeFilter.setEndAt(timeFormat.format(endTime));
|
@@ -196,9 +196,12 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
196
196
|
return marketoBulkExtractRequest;
|
197
197
|
}
|
198
198
|
|
199
|
-
public String createActivityExtract(Date startTime, Date endTime)
|
199
|
+
public String createActivityExtract(List<Integer> activityTypeIds, Date startTime, Date endTime)
|
200
200
|
{
|
201
201
|
MarketoBulkExtractRequest marketoBulkExtractRequest = getMarketoBulkExtractRequest(startTime, endTime, null, "createdAt");
|
202
|
+
if (activityTypeIds != null && !activityTypeIds.isEmpty()) {
|
203
|
+
marketoBulkExtractRequest.getFilter().put("activityTypeIds", activityTypeIds);
|
204
|
+
}
|
202
205
|
return sendCreateBulkExtractRequest(marketoBulkExtractRequest, MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT);
|
203
206
|
}
|
204
207
|
|
@@ -442,7 +445,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
442
445
|
// put filter params if exist.
|
443
446
|
if (filterType != null) {
|
444
447
|
multimap.put("filterType", filterType);
|
445
|
-
multimap.put("filterValues",
|
448
|
+
multimap.put("filterValues", StringUtils.join(filterValues, ","));
|
446
449
|
}
|
447
450
|
return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), multimap, ObjectNode.class);
|
448
451
|
}
|
@@ -534,4 +537,9 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
534
537
|
{
|
535
538
|
return getCustomObjectRecordWithPagination(endPoint + MarketoRESTEndpoint.GET_CUSTOM_OBJECT.getEndpoint(new ImmutableMap.Builder().put("api_name", customObjectAPIName).build()), customObjectFilterType, customObjectFields, fromValue, toValue, ObjectNode.class);
|
536
539
|
}
|
540
|
+
|
541
|
+
public Iterable<ObjectNode> getActivityTypes()
|
542
|
+
{
|
543
|
+
return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_ACTIVITY_TYPES.getEndpoint(), new ImmutableListMultimap.Builder<String, String>().put(MAX_RETURN, DEFAULT_MAX_RETURN).build(), ObjectNode.class);
|
544
|
+
}
|
537
545
|
}
|
@@ -22,6 +22,8 @@ import java.util.Date;
|
|
22
22
|
import java.util.Iterator;
|
23
23
|
import java.util.List;
|
24
24
|
|
25
|
+
import static org.mockito.ArgumentMatchers.any;
|
26
|
+
|
25
27
|
/**
|
26
28
|
* Created by tai.khuu on 10/9/17.
|
27
29
|
*/
|
@@ -45,12 +47,12 @@ public class MarketoServiceImplTest
|
|
45
47
|
{
|
46
48
|
Date startDate = new Date(1507223374000L);
|
47
49
|
Date endDate = new Date(1507655374000L);
|
48
|
-
List<String> extractedFields = Arrays.asList("field1", "
|
50
|
+
List<String> extractedFields = Arrays.asList("field1", "fActivityBulkExtractInputPluginTest.java:78ield2");
|
49
51
|
String filerField = "field1";
|
50
52
|
String exportId = "exportId";
|
51
53
|
Mockito.when(mockMarketoRestClient.createLeadBulkExtract(Mockito.eq(startDate), Mockito.eq(endDate), Mockito.eq(extractedFields), Mockito.eq(filerField))).thenReturn(exportId);
|
52
54
|
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
|
53
|
-
Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId),
|
55
|
+
Mockito.when(mockMarketoRestClient.getLeadBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
|
54
56
|
File file = marketoService.extractLead(startDate, endDate, extractedFields, filerField, 1, 3);
|
55
57
|
Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
|
56
58
|
Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startLeadBulkExtract(Mockito.eq(exportId));
|
@@ -62,11 +64,12 @@ public class MarketoServiceImplTest
|
|
62
64
|
{
|
63
65
|
Date startDate = new Date(1507223374000L);
|
64
66
|
Date endDate = new Date(1507655374000L);
|
67
|
+
List<Integer> activityTypeIds = new ArrayList<>();
|
65
68
|
String exportId = "exportId";
|
66
|
-
Mockito.when(mockMarketoRestClient.createActivityExtract(Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
|
69
|
+
Mockito.when(mockMarketoRestClient.createActivityExtract(any(List.class), Mockito.eq(startDate), Mockito.eq(endDate))).thenReturn(exportId);
|
67
70
|
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("Test File Content".getBytes());
|
68
|
-
Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId),
|
69
|
-
File file = marketoService.extractAllActivity(startDate, endDate, 1, 3);
|
71
|
+
Mockito.when(mockMarketoRestClient.getActivitiesBulkExtractResult(Mockito.eq(exportId), any(BulkExtractRangeHeader.class))).thenReturn(byteArrayInputStream);
|
72
|
+
File file = marketoService.extractAllActivity(activityTypeIds, startDate, endDate, 1, 3);
|
70
73
|
Assert.assertEquals("Test File Content", new String(ByteStreams.toByteArray(new FileInputStream(file))));
|
71
74
|
Mockito.verify(mockMarketoRestClient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId));
|
72
75
|
Mockito.verify(mockMarketoRestClient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId), Mockito.eq(1), Mockito.eq(3));
|
data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java
CHANGED
@@ -1,16 +1,22 @@
|
|
1
1
|
package org.embulk.input.marketo.delegate;
|
2
2
|
|
3
|
+
import com.fasterxml.jackson.databind.JavaType;
|
4
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
3
6
|
import org.embulk.EmbulkTestRuntime;
|
4
7
|
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
8
|
import org.embulk.base.restclient.record.ValueLocator;
|
6
9
|
import org.embulk.config.ConfigDiff;
|
10
|
+
import org.embulk.config.ConfigException;
|
7
11
|
import org.embulk.config.ConfigLoader;
|
8
12
|
import org.embulk.config.ConfigSource;
|
9
13
|
import org.embulk.config.TaskReport;
|
10
14
|
import org.embulk.input.marketo.MarketoUtils;
|
11
15
|
import org.embulk.input.marketo.model.BulkExtractRangeHeader;
|
12
16
|
import org.embulk.input.marketo.rest.MarketoRestClient;
|
17
|
+
import org.embulk.input.marketo.rest.RecordPagingIterable;
|
13
18
|
import org.embulk.spi.Column;
|
19
|
+
import org.embulk.spi.Exec;
|
14
20
|
import org.embulk.spi.PageBuilder;
|
15
21
|
import org.embulk.spi.Schema;
|
16
22
|
import org.joda.time.DateTime;
|
@@ -18,14 +24,17 @@ import org.junit.Assert;
|
|
18
24
|
import org.junit.Before;
|
19
25
|
import org.junit.Rule;
|
20
26
|
import org.junit.Test;
|
27
|
+
import org.junit.function.ThrowingRunnable;
|
21
28
|
import org.mockito.ArgumentCaptor;
|
22
29
|
import org.mockito.Mockito;
|
23
30
|
|
24
31
|
import java.io.IOException;
|
25
32
|
import java.text.DateFormat;
|
26
33
|
import java.text.SimpleDateFormat;
|
34
|
+
import java.util.ArrayList;
|
27
35
|
import java.util.Arrays;
|
28
36
|
import java.util.Date;
|
37
|
+
import java.util.List;
|
29
38
|
|
30
39
|
import static org.mockito.ArgumentMatchers.any;
|
31
40
|
|
@@ -34,6 +43,8 @@ import static org.mockito.ArgumentMatchers.any;
|
|
34
43
|
*/
|
35
44
|
public class ActivityBulkExtractInputPluginTest
|
36
45
|
{
|
46
|
+
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
47
|
+
|
37
48
|
@Rule
|
38
49
|
public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
|
39
50
|
|
@@ -53,15 +64,49 @@ public class ActivityBulkExtractInputPluginTest
|
|
53
64
|
Mockito.doReturn(mockMarketoRestclient).when(activityBulkExtractInputPlugin).createMarketoRestClient(any(ActivityBulkExtractInputPlugin.PluginTask.class));
|
54
65
|
}
|
55
66
|
|
67
|
+
@Test
|
68
|
+
public void testInvalidActivityTypeIds()
|
69
|
+
{
|
70
|
+
configSource.set("activity_type_ids", Arrays.asList(" ", "abc", "123"));
|
71
|
+
final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
|
72
|
+
|
73
|
+
Assert.assertThrows("Invalid activity type id: [ , abc]", ConfigException.class, new ThrowingRunnable()
|
74
|
+
{
|
75
|
+
@Override
|
76
|
+
public void run() throws Throwable
|
77
|
+
{
|
78
|
+
activityBulkExtractInputPlugin.validateInputTask(task);
|
79
|
+
}
|
80
|
+
});
|
81
|
+
}
|
82
|
+
|
83
|
+
@Test
|
84
|
+
public void testActivityTypeIdsValid() throws IOException
|
85
|
+
{
|
86
|
+
configSource.set("activity_type_ids", Arrays.asList("1", "2", "3 "));
|
87
|
+
final ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
|
88
|
+
|
89
|
+
RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
|
90
|
+
JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
|
91
|
+
List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/activity_types.json"), javaType);
|
92
|
+
Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
|
93
|
+
|
94
|
+
Mockito.when(mockMarketoRestclient.getActivityTypes()).thenReturn(mockRecordPagingIterable);
|
95
|
+
activityBulkExtractInputPlugin.validateInputTask(task);
|
96
|
+
}
|
97
|
+
|
56
98
|
@Test
|
57
99
|
public void testRun() throws InterruptedException
|
58
100
|
{
|
59
101
|
ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
|
102
|
+
|
60
103
|
DateTime startDate = new DateTime(task.getFromDate());
|
104
|
+
List<Integer> activityTypeIds = new ArrayList<>();
|
105
|
+
|
61
106
|
PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
|
62
107
|
String exportId1 = "exportId1";
|
63
108
|
String exportId2 = "exportId2";
|
64
|
-
Mockito.when(mockMarketoRestclient.createActivityExtract(any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
|
109
|
+
Mockito.when(mockMarketoRestclient.createActivityExtract(any(List.class), any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
|
65
110
|
Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract1.csv"));
|
66
111
|
Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract2.csv"));
|
67
112
|
ServiceResponseMapper<? extends ValueLocator> mapper = activityBulkExtractInputPlugin.buildServiceResponseMapper(task);
|
@@ -74,9 +119,9 @@ public class ActivityBulkExtractInputPluginTest
|
|
74
119
|
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId1), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
|
75
120
|
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId2));
|
76
121
|
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId2), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
|
77
|
-
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
|
122
|
+
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate.toDate(), startDate.plusDays(30).toDate());
|
78
123
|
DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
|
79
|
-
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
|
124
|
+
Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(activityTypeIds, startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
|
80
125
|
ConfigDiff configDiff = activityBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
|
81
126
|
DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
|
82
127
|
Assert.assertEquals(df.format(startDate.plusDays(task.getFetchDays()).toDate()), configDiff.get(String.class, "from_date"));
|
@@ -35,6 +35,7 @@ import java.io.IOException;
|
|
35
35
|
import java.io.InputStream;
|
36
36
|
import java.nio.ByteBuffer;
|
37
37
|
import java.text.SimpleDateFormat;
|
38
|
+
import java.util.ArrayList;
|
38
39
|
import java.util.Arrays;
|
39
40
|
import java.util.Date;
|
40
41
|
import java.util.HashMap;
|
@@ -149,12 +150,13 @@ public class MarketoRestClientTest
|
|
149
150
|
marketoResponse.setSuccess(true);
|
150
151
|
Date startDate = new Date(1506865856000L);
|
151
152
|
Date endDate = new Date(1507297856000L);
|
153
|
+
List<Integer> activityTypeIds = new ArrayList<>();
|
152
154
|
ObjectNode bulkExtractResult = OBJECT_MAPPER.createObjectNode();
|
153
155
|
bulkExtractResult.set("exportId", new TextNode("bulkExtractId"));
|
154
156
|
marketoResponse.setResult(Arrays.asList(bulkExtractResult));
|
155
157
|
ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
|
156
158
|
Mockito.doReturn(marketoResponse).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_ACTIVITY_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), argumentCaptor.capture(), Mockito.any(MarketoResponseJetty92EntityReader.class));
|
157
|
-
String bulkExtractId = marketoRestClient.createActivityExtract(startDate, endDate);
|
159
|
+
String bulkExtractId = marketoRestClient.createActivityExtract(activityTypeIds, startDate, endDate);
|
158
160
|
Assert.assertEquals("bulkExtractId", bulkExtractId);
|
159
161
|
String postContent = argumentCaptor.getValue();
|
160
162
|
ObjectNode marketoBulkExtractRequest = (ObjectNode) OBJECT_MAPPER.readTree(postContent);
|
@@ -0,0 +1,22 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"id": 1,
|
4
|
+
"name": "bb"
|
5
|
+
},
|
6
|
+
{
|
7
|
+
"id": 2,
|
8
|
+
"name": "TD Output Test aa"
|
9
|
+
},
|
10
|
+
{
|
11
|
+
"id": 3,
|
12
|
+
"name": "Bill_progream a"
|
13
|
+
},
|
14
|
+
{
|
15
|
+
"id": 4,
|
16
|
+
"name": "Bill_progream b"
|
17
|
+
},
|
18
|
+
{
|
19
|
+
"id": 5,
|
20
|
+
"name": "Bill_progream c"
|
21
|
+
}
|
22
|
+
]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2019-
|
13
|
+
date: 2019-12-03 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,7 +89,6 @@ files:
|
|
89
89
|
- src/main/java/org/embulk/input/marketo/model/MarketoField.java
|
90
90
|
- src/main/java/org/embulk/input/marketo/model/MarketoResponse.java
|
91
91
|
- src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java
|
92
|
-
- src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java
|
93
92
|
- src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java
|
94
93
|
- src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java
|
95
94
|
- src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java
|
@@ -115,6 +114,7 @@ files:
|
|
115
114
|
- src/test/resources/config/rest_config.yaml
|
116
115
|
- src/test/resources/fixtures/activity_extract1.csv
|
117
116
|
- src/test/resources/fixtures/activity_extract2.csv
|
117
|
+
- src/test/resources/fixtures/activity_types.json
|
118
118
|
- src/test/resources/fixtures/all_program_full.json
|
119
119
|
- src/test/resources/fixtures/campaign_response.json
|
120
120
|
- src/test/resources/fixtures/campaign_response_full.json
|
@@ -137,9 +137,9 @@ files:
|
|
137
137
|
- src/test/resources/fixtures/program_response.json
|
138
138
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
139
139
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
140
|
-
- classpath/embulk-input-marketo-0.6.15.jar
|
141
140
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
142
141
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
142
|
+
- classpath/embulk-input-marketo-0.6.16.jar
|
143
143
|
- classpath/embulk-base-restclient-0.5.3.jar
|
144
144
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
145
145
|
homepage: https://github.com/treasure-data/embulk-input-marketo
|