embulk-input-marketo-extended 0.6.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
- data/.gitignore +14 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +170 -0
- data/LICENSE.txt +21 -0
- data/README.md +213 -0
- data/build.gradle +103 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +169 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/marketo.rb +3 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
- data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
- data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
- data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
- data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
- data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
- data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
- data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
- data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
- data/src/test/resources/config/custom_object_config.yaml +8 -0
- data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
- data/src/test/resources/config/rest_config.yaml +3 -0
- data/src/test/resources/fixtures/activity_extract1.csv +35 -0
- data/src/test/resources/fixtures/activity_extract2.csv +22 -0
- data/src/test/resources/fixtures/activity_types.json +22 -0
- data/src/test/resources/fixtures/all_program_full.json +53 -0
- data/src/test/resources/fixtures/campaign_response.json +38 -0
- data/src/test/resources/fixtures/campaign_response_full.json +102 -0
- data/src/test/resources/fixtures/custom_object_describe.json +124 -0
- data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
- data/src/test/resources/fixtures/custom_object_expected.json +66 -0
- data/src/test/resources/fixtures/custom_object_response.json +24 -0
- data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
- data/src/test/resources/fixtures/lead_by_list.json +33 -0
- data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
- data/src/test/resources/fixtures/lead_describe.json +221 -0
- data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
- data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
- data/src/test/resources/fixtures/lead_extract1.csv +11 -0
- data/src/test/resources/fixtures/lead_response_full.json +2402 -0
- data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
- data/src/test/resources/fixtures/leads_extract2.csv +10 -0
- data/src/test/resources/fixtures/list_reponse_full.json +191 -0
- data/src/test/resources/fixtures/lists_response.json +31 -0
- data/src/test/resources/fixtures/program_response.json +71 -0
- metadata +171 -0
@@ -0,0 +1,167 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
|
+
import com.google.common.annotations.VisibleForTesting;
|
5
|
+
import com.google.common.base.Optional;
|
6
|
+
import org.apache.commons.lang3.StringUtils;
|
7
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
8
|
+
import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
|
9
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
10
|
+
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
12
|
+
import org.embulk.config.ConfigException;
|
13
|
+
import org.embulk.input.marketo.MarketoService;
|
14
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
15
|
+
import org.embulk.input.marketo.MarketoUtils;
|
16
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
17
|
+
import org.embulk.spi.type.Types;
|
18
|
+
import org.joda.time.DateTime;
|
19
|
+
|
20
|
+
import java.io.FileInputStream;
|
21
|
+
import java.io.FileNotFoundException;
|
22
|
+
import java.io.InputStream;
|
23
|
+
import java.text.MessageFormat;
|
24
|
+
import java.util.ArrayList;
|
25
|
+
import java.util.Iterator;
|
26
|
+
import java.util.List;
|
27
|
+
|
28
|
+
/**
|
29
|
+
* Created by tai.khuu on 9/18/17.
|
30
|
+
*/
|
31
|
+
public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
|
32
|
+
{
|
33
|
+
public static final String INCREMENTAL_COLUMN = "activityDate";
|
34
|
+
public static final String UID_COLUMN = "marketoGUID";
|
35
|
+
|
36
|
+
public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
|
37
|
+
{
|
38
|
+
@Config("activity_type_ids")
|
39
|
+
@ConfigDefault("[]")
|
40
|
+
List<String> getActivityTypeIds();
|
41
|
+
|
42
|
+
@Config("act_type_ids")
|
43
|
+
@ConfigDefault("[]")
|
44
|
+
List<Integer> getActTypeIds();
|
45
|
+
|
46
|
+
void setActTypeIds(List<Integer> activityIds);
|
47
|
+
}
|
48
|
+
|
49
|
+
@Override
|
50
|
+
public void validateInputTask(PluginTask task)
|
51
|
+
{
|
52
|
+
task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
|
53
|
+
task.setUidColumn(Optional.of(UID_COLUMN));
|
54
|
+
if (!task.getActivityTypeIds().isEmpty()) {
|
55
|
+
List<Integer> activityIds = checkValidActivityTypeIds(task);
|
56
|
+
|
57
|
+
// check input with values from server
|
58
|
+
try (MarketoRestClient restClient = createMarketoRestClient(task)) {
|
59
|
+
MarketoService marketoService = new MarketoServiceImpl(restClient);
|
60
|
+
Iterable<ObjectNode> nodes = marketoService.getActivityTypes();
|
61
|
+
if (nodes != null) {
|
62
|
+
checkValidActivityTypeIds(nodes, activityIds);
|
63
|
+
}
|
64
|
+
// ignorable if unable to get activity type ids. If thing gone wrong, the bulk extract will throw errors
|
65
|
+
}
|
66
|
+
|
67
|
+
// task will use getActTypeIds instead of getActivityTypeIds method
|
68
|
+
task.setActTypeIds(activityIds);
|
69
|
+
}
|
70
|
+
super.validateInputTask(task);
|
71
|
+
}
|
72
|
+
|
73
|
+
/**
|
74
|
+
* Check if user input activity_type_ids valid
|
75
|
+
* @param task
|
76
|
+
* @return values transformed to array of Integer
|
77
|
+
*/
|
78
|
+
private List<Integer> checkValidActivityTypeIds(PluginTask task)
|
79
|
+
{
|
80
|
+
List<String> invalidIds = new ArrayList<>();
|
81
|
+
for (String id : task.getActivityTypeIds()) {
|
82
|
+
if (StringUtils.isBlank(id) || !StringUtils.isNumeric(StringUtils.trimToEmpty(id))) {
|
83
|
+
invalidIds.add(id);
|
84
|
+
}
|
85
|
+
}
|
86
|
+
|
87
|
+
if (!invalidIds.isEmpty()) {
|
88
|
+
throw new ConfigException(MessageFormat.format("Invalid activity type id: [{0}]", StringUtils.join(invalidIds, ", ")));
|
89
|
+
}
|
90
|
+
|
91
|
+
// transform and set
|
92
|
+
List<Integer> activityIds = new ArrayList<>();
|
93
|
+
for (String id : task.getActivityTypeIds()) {
|
94
|
+
activityIds.add(Integer.valueOf(StringUtils.trimToEmpty(id)));
|
95
|
+
}
|
96
|
+
|
97
|
+
return activityIds;
|
98
|
+
}
|
99
|
+
|
100
|
+
@VisibleForTesting
|
101
|
+
protected void checkValidActivityTypeIds(Iterable<ObjectNode> nodes, List<Integer> activityIds)
|
102
|
+
{
|
103
|
+
Iterator<ObjectNode> it = nodes.iterator();
|
104
|
+
|
105
|
+
List<Integer> inputIds = new ArrayList<>(activityIds);
|
106
|
+
|
107
|
+
while (it.hasNext()) {
|
108
|
+
ObjectNode node = it.next();
|
109
|
+
int id = node.get("id").asInt(0);
|
110
|
+
if (id > 0) {
|
111
|
+
inputIds.remove(Integer.valueOf(id));
|
112
|
+
}
|
113
|
+
}
|
114
|
+
|
115
|
+
if (!inputIds.isEmpty()) {
|
116
|
+
throw new ConfigException(MessageFormat.format("Invalid activity type ids: [{0}], Available activity types: \n{1}",
|
117
|
+
StringUtils.join(inputIds, ", "),
|
118
|
+
buildActivityIdNameInfo(nodes)));
|
119
|
+
}
|
120
|
+
}
|
121
|
+
|
122
|
+
private String buildActivityIdNameInfo(Iterable<ObjectNode> nodes)
|
123
|
+
{
|
124
|
+
Iterator<ObjectNode> it = nodes.iterator();
|
125
|
+
StringBuilder messageBuilder = new StringBuilder();
|
126
|
+
while (it.hasNext()) {
|
127
|
+
ObjectNode node = it.next();
|
128
|
+
int id = node.get("id").asInt(0);
|
129
|
+
String name = node.get("name").asText("");
|
130
|
+
if (id > 0) {
|
131
|
+
messageBuilder.append("- activity id: ");
|
132
|
+
messageBuilder.append(String.valueOf(id));
|
133
|
+
messageBuilder.append(", name: ");
|
134
|
+
messageBuilder.append(name);
|
135
|
+
messageBuilder.append("\n");
|
136
|
+
}
|
137
|
+
}
|
138
|
+
|
139
|
+
return messageBuilder.toString();
|
140
|
+
}
|
141
|
+
|
142
|
+
@Override
|
143
|
+
protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
|
144
|
+
{
|
145
|
+
try {
|
146
|
+
return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
|
147
|
+
}
|
148
|
+
catch (FileNotFoundException e) {
|
149
|
+
throw new RuntimeException("Exception when trying to extract activity", e);
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
@Override
|
154
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
155
|
+
{
|
156
|
+
JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
|
157
|
+
builder.add("marketoGUID", Types.STRING)
|
158
|
+
.add("leadId", Types.STRING)
|
159
|
+
.add("activityDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
|
160
|
+
.add("activityTypeId", Types.STRING)
|
161
|
+
.add("campaignId", Types.STRING)
|
162
|
+
.add("primaryAttributeValueId", Types.STRING)
|
163
|
+
.add("primaryAttributeValue", Types.STRING)
|
164
|
+
.add("attributes", Types.JSON);
|
165
|
+
return builder.build();
|
166
|
+
}
|
167
|
+
}
|
@@ -0,0 +1,48 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.collect.FluentIterable;
|
4
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
|
+
import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
|
6
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
7
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
8
|
+
import org.embulk.input.marketo.MarketoService;
|
9
|
+
import org.embulk.input.marketo.MarketoUtils;
|
10
|
+
import org.embulk.spi.type.Types;
|
11
|
+
|
12
|
+
import java.util.Iterator;
|
13
|
+
|
14
|
+
/**
|
15
|
+
* Input plugin use to import Campaign
|
16
|
+
* Created by tai.khuu on 9/18/17.
|
17
|
+
*/
|
18
|
+
public class CampaignInputPlugin extends MarketoBaseInputPluginDelegate<CampaignInputPlugin.PluginTask>
|
19
|
+
{
|
20
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
|
21
|
+
{
|
22
|
+
}
|
23
|
+
|
24
|
+
public CampaignInputPlugin()
|
25
|
+
{
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
|
30
|
+
{
|
31
|
+
return FluentIterable.from(marketoService.getCampaign()).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
|
32
|
+
}
|
33
|
+
|
34
|
+
@Override
|
35
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
36
|
+
{
|
37
|
+
JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
|
38
|
+
builder.add("id", Types.LONG)
|
39
|
+
.add("name", Types.STRING)
|
40
|
+
.add("description", Types.STRING)
|
41
|
+
.add("type", Types.STRING)
|
42
|
+
.add("workspaceName", Types.STRING)
|
43
|
+
.add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
|
44
|
+
.add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
|
45
|
+
.add("active", Types.BOOLEAN);
|
46
|
+
return builder.build();
|
47
|
+
}
|
48
|
+
}
|
@@ -0,0 +1,75 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import com.google.common.collect.FluentIterable;
|
5
|
+
import org.apache.commons.lang3.StringUtils;
|
6
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
7
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
8
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
9
|
+
import org.embulk.config.Config;
|
10
|
+
import org.embulk.config.ConfigDefault;
|
11
|
+
import org.embulk.config.ConfigException;
|
12
|
+
import org.embulk.input.marketo.MarketoService;
|
13
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
14
|
+
import org.embulk.input.marketo.MarketoUtils;
|
15
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
16
|
+
import org.embulk.spi.Exec;
|
17
|
+
import org.slf4j.Logger;
|
18
|
+
|
19
|
+
import java.util.Iterator;
|
20
|
+
|
21
|
+
public class CustomObjectInputPlugin extends MarketoBaseInputPluginDelegate<CustomObjectInputPlugin.PluginTask>
|
22
|
+
{
|
23
|
+
private final Logger logger = Exec.getLogger(getClass());
|
24
|
+
|
25
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CustomObjectResponseMapperBuilder.PluginTask
|
26
|
+
{
|
27
|
+
@Config("custom_object_filter_type")
|
28
|
+
@ConfigDefault("\"\"")
|
29
|
+
String getCustomObjectFilterType();
|
30
|
+
|
31
|
+
@Config("custom_object_filter_from_value")
|
32
|
+
@ConfigDefault("1")
|
33
|
+
Integer getFromValue();
|
34
|
+
|
35
|
+
@Config("custom_object_filter_to_value")
|
36
|
+
@ConfigDefault("null")
|
37
|
+
Optional<Integer> getToValue();
|
38
|
+
}
|
39
|
+
|
40
|
+
public CustomObjectInputPlugin()
|
41
|
+
{
|
42
|
+
}
|
43
|
+
|
44
|
+
@Override
|
45
|
+
public void validateInputTask(PluginTask task)
|
46
|
+
{
|
47
|
+
super.validateInputTask(task);
|
48
|
+
if (StringUtils.isBlank(task.getCustomObjectFilterType())) {
|
49
|
+
throw new ConfigException("`custom_object_filter_type` cannot be empty");
|
50
|
+
}
|
51
|
+
if (StringUtils.isBlank(task.getCustomObjectAPIName())) {
|
52
|
+
throw new ConfigException("`custom_object_api_name` cannot be empty");
|
53
|
+
}
|
54
|
+
if (task.getToValue().isPresent() && task.getToValue().get() < task.getFromValue()) {
|
55
|
+
throw new ConfigException(String.format("`to_value` (%s) cannot be less than the `from_value` (%s)", task.getToValue().get(), task.getFromValue()));
|
56
|
+
}
|
57
|
+
}
|
58
|
+
@Override
|
59
|
+
protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
|
60
|
+
{
|
61
|
+
return FluentIterable
|
62
|
+
.from(marketoService.getCustomObject(task.getCustomObjectAPIName(), task.getCustomObjectFilterType(), task.getCustomObjectFields().orNull(), task.getFromValue(), task.getToValue().orNull()))
|
63
|
+
.transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
|
64
|
+
}
|
65
|
+
|
66
|
+
@Override
|
67
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
68
|
+
{
|
69
|
+
try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
|
70
|
+
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
71
|
+
CustomObjectResponseMapperBuilder<PluginTask> customObjectResponseMapperBuilder = new CustomObjectResponseMapperBuilder<>(task, marketoService);
|
72
|
+
return customObjectResponseMapperBuilder.buildServiceResponseMapper(task);
|
73
|
+
}
|
74
|
+
}
|
75
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.apache.commons.lang3.StringUtils;
|
5
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
6
|
+
import org.embulk.base.restclient.ServiceResponseMapperBuildable;
|
7
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
8
|
+
import org.embulk.config.Config;
|
9
|
+
import org.embulk.config.ConfigDefault;
|
10
|
+
import org.embulk.input.marketo.MarketoService;
|
11
|
+
import org.embulk.input.marketo.MarketoUtils;
|
12
|
+
import org.embulk.input.marketo.model.MarketoField;
|
13
|
+
import org.embulk.spi.Exec;
|
14
|
+
import org.slf4j.Logger;
|
15
|
+
|
16
|
+
import java.util.ArrayList;
|
17
|
+
import java.util.Arrays;
|
18
|
+
import java.util.List;
|
19
|
+
|
20
|
+
public class CustomObjectResponseMapperBuilder<T extends CustomObjectResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
|
21
|
+
{
|
22
|
+
private static final Logger LOGGER = Exec.getLogger(CustomObjectResponseMapperBuilder.class);
|
23
|
+
private MarketoService marketoService;
|
24
|
+
|
25
|
+
private T pluginTask;
|
26
|
+
|
27
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
|
28
|
+
{
|
29
|
+
@Config("custom_object_api_name")
|
30
|
+
@ConfigDefault("\"\"")
|
31
|
+
String getCustomObjectAPIName();
|
32
|
+
|
33
|
+
@Config("custom_object_fields")
|
34
|
+
@ConfigDefault("null")
|
35
|
+
Optional<String> getCustomObjectFields();
|
36
|
+
}
|
37
|
+
|
38
|
+
public CustomObjectResponseMapperBuilder(T task, MarketoService marketoService)
|
39
|
+
{
|
40
|
+
this.pluginTask = task;
|
41
|
+
this.marketoService = marketoService;
|
42
|
+
}
|
43
|
+
|
44
|
+
protected List<MarketoField> getCustomObjectColumns()
|
45
|
+
{
|
46
|
+
List<MarketoField> columns = marketoService.describeCustomObject(pluginTask.getCustomObjectAPIName());
|
47
|
+
if (pluginTask.getCustomObjectFields().isPresent() && StringUtils.isNotBlank(pluginTask.getCustomObjectFields().get())) {
|
48
|
+
List<MarketoField> filteredColumns = new ArrayList<>();
|
49
|
+
List<String> includedFields = Arrays.asList(pluginTask.getCustomObjectFields().get().split(","));
|
50
|
+
for (String fieldName : includedFields) {
|
51
|
+
Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
|
52
|
+
if (includedField.isPresent()) {
|
53
|
+
filteredColumns.add(includedField.get());
|
54
|
+
}
|
55
|
+
else {
|
56
|
+
LOGGER.warn("Included field [{}] not found in Marketo Custom Object [{}] field", fieldName, pluginTask.getCustomObjectAPIName());
|
57
|
+
}
|
58
|
+
}
|
59
|
+
columns = filteredColumns;
|
60
|
+
LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
|
61
|
+
}
|
62
|
+
return columns;
|
63
|
+
}
|
64
|
+
|
65
|
+
private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
|
66
|
+
{
|
67
|
+
for (MarketoField marketoField : inputList) {
|
68
|
+
if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
|
69
|
+
return Optional.of(marketoField);
|
70
|
+
}
|
71
|
+
}
|
72
|
+
return Optional.absent();
|
73
|
+
}
|
74
|
+
|
75
|
+
@Override
|
76
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
|
77
|
+
{
|
78
|
+
List<MarketoField> customObjectColumns = getCustomObjectColumns();
|
79
|
+
return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), customObjectColumns);
|
80
|
+
}
|
81
|
+
}
|
@@ -0,0 +1,66 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
6
|
+
import org.embulk.config.Config;
|
7
|
+
import org.embulk.config.ConfigDefault;
|
8
|
+
import org.embulk.input.marketo.MarketoService;
|
9
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
10
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
11
|
+
import org.embulk.spi.Exec;
|
12
|
+
import org.joda.time.DateTime;
|
13
|
+
import org.slf4j.Logger;
|
14
|
+
|
15
|
+
import java.io.FileInputStream;
|
16
|
+
import java.io.FileNotFoundException;
|
17
|
+
import java.io.InputStream;
|
18
|
+
import java.util.List;
|
19
|
+
|
20
|
+
/**
|
21
|
+
* Created by tai.khuu on 9/18/17.
|
22
|
+
*/
|
23
|
+
public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<LeadBulkExtractInputPlugin.PluginTask>
|
24
|
+
{
|
25
|
+
private static final Logger LOGGER = Exec.getLogger(LeadBulkExtractInputPlugin.class);
|
26
|
+
|
27
|
+
private static final String UPDATED_AT = "updatedAt";
|
28
|
+
|
29
|
+
public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
|
30
|
+
{
|
31
|
+
@Config("use_updated_at")
|
32
|
+
@ConfigDefault("false")
|
33
|
+
boolean getUseUpdatedAt();
|
34
|
+
}
|
35
|
+
|
36
|
+
@Override
|
37
|
+
public void validateInputTask(PluginTask task)
|
38
|
+
{
|
39
|
+
if (task.getUseUpdatedAt()) {
|
40
|
+
task.setIncrementalColumn(Optional.of(UPDATED_AT));
|
41
|
+
}
|
42
|
+
super.validateInputTask(task);
|
43
|
+
}
|
44
|
+
|
45
|
+
@Override
|
46
|
+
protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
|
47
|
+
{
|
48
|
+
try {
|
49
|
+
List<String> fieldNames = task.getExtractedFields();
|
50
|
+
return new FileInputStream(service.extractLead(fromDate.toDate(), toDate.toDate(), fieldNames, task.getIncrementalColumn().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
|
51
|
+
}
|
52
|
+
catch (FileNotFoundException e) {
|
53
|
+
throw new RuntimeException("File not found", e);
|
54
|
+
}
|
55
|
+
}
|
56
|
+
|
57
|
+
@Override
|
58
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
59
|
+
{
|
60
|
+
try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
|
61
|
+
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
62
|
+
LeadServiceResponseMapperBuilder<PluginTask> leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(task, marketoService);
|
63
|
+
return leadServiceResponseMapperBuilder.buildServiceResponseMapper(task);
|
64
|
+
}
|
65
|
+
}
|
66
|
+
}
|
@@ -0,0 +1,85 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
|
+
import org.embulk.base.restclient.ServiceResponseMapperBuildable;
|
6
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
7
|
+
import org.embulk.config.Config;
|
8
|
+
import org.embulk.config.ConfigDefault;
|
9
|
+
import org.embulk.input.marketo.MarketoService;
|
10
|
+
import org.embulk.input.marketo.MarketoUtils;
|
11
|
+
import org.embulk.input.marketo.model.MarketoField;
|
12
|
+
import org.embulk.spi.Exec;
|
13
|
+
import org.slf4j.Logger;
|
14
|
+
|
15
|
+
import java.util.ArrayList;
|
16
|
+
import java.util.List;
|
17
|
+
|
18
|
+
/**
|
19
|
+
* Created by tai.khuu on 5/21/18.
|
20
|
+
*/
|
21
|
+
public class LeadServiceResponseMapperBuilder<T extends LeadServiceResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
|
22
|
+
{
|
23
|
+
private static final Logger LOGGER = Exec.getLogger(LeadServiceResponseMapperBuilder.class);
|
24
|
+
private MarketoService marketoService;
|
25
|
+
|
26
|
+
private T pluginTask;
|
27
|
+
|
28
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
|
29
|
+
{
|
30
|
+
@Config("included_fields")
|
31
|
+
@ConfigDefault("null")
|
32
|
+
Optional<List<String>> getIncludedLeadFields();
|
33
|
+
|
34
|
+
@Config("extracted_fields")
|
35
|
+
@ConfigDefault("[]")
|
36
|
+
List<String> getExtractedFields();
|
37
|
+
|
38
|
+
void setExtractedFields(List<String> extractedFields);
|
39
|
+
}
|
40
|
+
|
41
|
+
public LeadServiceResponseMapperBuilder(T task, MarketoService marketoService)
|
42
|
+
{
|
43
|
+
this.pluginTask = task;
|
44
|
+
this.marketoService = marketoService;
|
45
|
+
}
|
46
|
+
|
47
|
+
protected List<MarketoField> getLeadColumns()
|
48
|
+
{
|
49
|
+
List<MarketoField> columns = marketoService.describeLead();
|
50
|
+
if (pluginTask.getIncludedLeadFields().isPresent() && !pluginTask.getIncludedLeadFields().get().isEmpty()) {
|
51
|
+
List<MarketoField> filteredColumns = new ArrayList<>();
|
52
|
+
List<String> includedFields = pluginTask.getIncludedLeadFields().get();
|
53
|
+
for (String fieldName : includedFields) {
|
54
|
+
Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
|
55
|
+
if (includedField.isPresent()) {
|
56
|
+
filteredColumns.add(includedField.get());
|
57
|
+
}
|
58
|
+
else {
|
59
|
+
LOGGER.warn("Included field [{}] not found in Marketo lead field", fieldName);
|
60
|
+
}
|
61
|
+
}
|
62
|
+
columns = filteredColumns;
|
63
|
+
LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
|
64
|
+
}
|
65
|
+
return columns;
|
66
|
+
}
|
67
|
+
|
68
|
+
private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
|
69
|
+
{
|
70
|
+
for (MarketoField marketoField : inputList) {
|
71
|
+
if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
|
72
|
+
return Optional.of(marketoField);
|
73
|
+
}
|
74
|
+
}
|
75
|
+
return Optional.absent();
|
76
|
+
}
|
77
|
+
|
78
|
+
@Override
|
79
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
|
80
|
+
{
|
81
|
+
List<MarketoField> leadColumns = getLeadColumns();
|
82
|
+
pluginTask.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(leadColumns));
|
83
|
+
return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), leadColumns);
|
84
|
+
}
|
85
|
+
}
|