embulk-input-marketo-extended 0.6.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
- data/.gitignore +14 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +170 -0
- data/LICENSE.txt +21 -0
- data/README.md +213 -0
- data/build.gradle +103 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +169 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/marketo.rb +3 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
- data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
- data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
- data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
- data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
- data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
- data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
- data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
- data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
- data/src/test/resources/config/custom_object_config.yaml +8 -0
- data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
- data/src/test/resources/config/rest_config.yaml +3 -0
- data/src/test/resources/fixtures/activity_extract1.csv +35 -0
- data/src/test/resources/fixtures/activity_extract2.csv +22 -0
- data/src/test/resources/fixtures/activity_types.json +22 -0
- data/src/test/resources/fixtures/all_program_full.json +53 -0
- data/src/test/resources/fixtures/campaign_response.json +38 -0
- data/src/test/resources/fixtures/campaign_response_full.json +102 -0
- data/src/test/resources/fixtures/custom_object_describe.json +124 -0
- data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
- data/src/test/resources/fixtures/custom_object_expected.json +66 -0
- data/src/test/resources/fixtures/custom_object_response.json +24 -0
- data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
- data/src/test/resources/fixtures/lead_by_list.json +33 -0
- data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
- data/src/test/resources/fixtures/lead_describe.json +221 -0
- data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
- data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
- data/src/test/resources/fixtures/lead_extract1.csv +11 -0
- data/src/test/resources/fixtures/lead_response_full.json +2402 -0
- data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
- data/src/test/resources/fixtures/leads_extract2.csv +10 -0
- data/src/test/resources/fixtures/list_reponse_full.json +191 -0
- data/src/test/resources/fixtures/lists_response.json +31 -0
- data/src/test/resources/fixtures/program_response.json +71 -0
- metadata +171 -0
@@ -0,0 +1,15 @@
|
|
1
|
+
package org.embulk.input.marketo;
|
2
|
+
|
3
|
+
import org.embulk.base.restclient.RestClientInputPluginBase;
|
4
|
+
|
5
|
+
/**
|
6
|
+
* Created by tai.khuu on 8/22/17.
|
7
|
+
*/
|
8
|
+
public class MarketoInputPlugin
|
9
|
+
extends RestClientInputPluginBase<MarketoInputPluginDelegate.PluginTask>
|
10
|
+
{
|
11
|
+
public MarketoInputPlugin()
|
12
|
+
{
|
13
|
+
super(MarketoInputPluginDelegate.PluginTask.class, new MarketoInputPluginDelegate());
|
14
|
+
}
|
15
|
+
}
|
@@ -0,0 +1,100 @@
|
|
1
|
+
package org.embulk.input.marketo;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonIgnore;
|
5
|
+
import com.google.common.base.Optional;
|
6
|
+
|
7
|
+
import org.embulk.base.restclient.DispatchingRestClientInputPluginDelegate;
|
8
|
+
import org.embulk.base.restclient.RestClientInputPluginDelegate;
|
9
|
+
import org.embulk.config.Config;
|
10
|
+
import org.embulk.config.ConfigDefault;
|
11
|
+
import org.embulk.config.ConfigException;
|
12
|
+
import org.embulk.input.marketo.delegate.ActivityBulkExtractInputPlugin;
|
13
|
+
import org.embulk.input.marketo.delegate.CampaignInputPlugin;
|
14
|
+
import org.embulk.input.marketo.delegate.CustomObjectInputPlugin;
|
15
|
+
import org.embulk.input.marketo.delegate.LeadBulkExtractInputPlugin;
|
16
|
+
import org.embulk.input.marketo.delegate.LeadWithListInputPlugin;
|
17
|
+
import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin;
|
18
|
+
import org.embulk.input.marketo.delegate.ProgramInputPlugin;
|
19
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
20
|
+
|
21
|
+
import java.util.Date;
|
22
|
+
|
23
|
+
public class MarketoInputPluginDelegate
|
24
|
+
extends DispatchingRestClientInputPluginDelegate<MarketoInputPluginDelegate.PluginTask>
|
25
|
+
{
|
26
|
+
public interface PluginTask
|
27
|
+
extends LeadWithListInputPlugin.PluginTask,
|
28
|
+
LeadBulkExtractInputPlugin.PluginTask,
|
29
|
+
LeadWithProgramInputPlugin.PluginTask,
|
30
|
+
ActivityBulkExtractInputPlugin.PluginTask,
|
31
|
+
CampaignInputPlugin.PluginTask,
|
32
|
+
ProgramInputPlugin.PluginTask,
|
33
|
+
MarketoRestClient.PluginTask,
|
34
|
+
CustomObjectInputPlugin.PluginTask
|
35
|
+
{
|
36
|
+
@Config("target")
|
37
|
+
Target getTarget();
|
38
|
+
|
39
|
+
//We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
|
40
|
+
//We will hide the real from_date, and set it when validating task
|
41
|
+
@Config("hidden_from_date")
|
42
|
+
@ConfigDefault("\"1970-01-01\"")
|
43
|
+
@Override
|
44
|
+
Date getFromDate();
|
45
|
+
|
46
|
+
void setFromDate(Date fromDate);
|
47
|
+
|
48
|
+
@Config("from_date")
|
49
|
+
@ConfigDefault("null")
|
50
|
+
Optional<Date> getWrappedFromDate();
|
51
|
+
}
|
52
|
+
|
53
|
+
@SuppressWarnings("unchecked")
|
54
|
+
@Override
|
55
|
+
protected RestClientInputPluginDelegate dispatchPerTask(PluginTask task)
|
56
|
+
{
|
57
|
+
Target target = task.getTarget();
|
58
|
+
switch (target) {
|
59
|
+
case LEAD:
|
60
|
+
case ACTIVITY:
|
61
|
+
if (!task.getWrappedFromDate().isPresent()) {
|
62
|
+
throw new ConfigException("From date is required for target LEAD or ACTIVITY");
|
63
|
+
}
|
64
|
+
Date date = task.getWrappedFromDate().get();
|
65
|
+
task.setFromDate(date);
|
66
|
+
break;
|
67
|
+
}
|
68
|
+
return target.getRestClientInputPluginDelegate();
|
69
|
+
}
|
70
|
+
|
71
|
+
public enum Target
|
72
|
+
{
|
73
|
+
LEAD(new LeadBulkExtractInputPlugin()),
|
74
|
+
ACTIVITY(new ActivityBulkExtractInputPlugin()),
|
75
|
+
CAMPAIGN(new CampaignInputPlugin()),
|
76
|
+
ALL_LEAD_WITH_LIST_ID(new LeadWithListInputPlugin()),
|
77
|
+
ALL_LEAD_WITH_PROGRAM_ID(new LeadWithProgramInputPlugin()),
|
78
|
+
PROGRAM(new ProgramInputPlugin()),
|
79
|
+
CUSTOM_OBJECT(new CustomObjectInputPlugin());
|
80
|
+
|
81
|
+
private RestClientInputPluginDelegate restClientInputPluginDelegate;
|
82
|
+
|
83
|
+
Target(RestClientInputPluginDelegate restClientInputPluginDelegate)
|
84
|
+
{
|
85
|
+
this.restClientInputPluginDelegate = restClientInputPluginDelegate;
|
86
|
+
}
|
87
|
+
|
88
|
+
@JsonIgnore
|
89
|
+
public RestClientInputPluginDelegate getRestClientInputPluginDelegate()
|
90
|
+
{
|
91
|
+
return restClientInputPluginDelegate;
|
92
|
+
}
|
93
|
+
|
94
|
+
@JsonCreator
|
95
|
+
public static Target of(String value)
|
96
|
+
{
|
97
|
+
return Target.valueOf(value.toUpperCase());
|
98
|
+
}
|
99
|
+
}
|
100
|
+
}
|
@@ -0,0 +1,38 @@
|
|
1
|
+
package org.embulk.input.marketo;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
|
+
import org.embulk.input.marketo.model.MarketoField;
|
5
|
+
|
6
|
+
import java.io.File;
|
7
|
+
import java.util.Date;
|
8
|
+
import java.util.List;
|
9
|
+
|
10
|
+
/**
|
11
|
+
* Created by tai.khuu on 9/6/17.
|
12
|
+
*/
|
13
|
+
public interface MarketoService
|
14
|
+
{
|
15
|
+
List<MarketoField> describeLead();
|
16
|
+
|
17
|
+
File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
|
18
|
+
|
19
|
+
File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
|
20
|
+
|
21
|
+
Iterable<ObjectNode> getAllListLead(List<String> extractFields);
|
22
|
+
|
23
|
+
Iterable<ObjectNode> getAllProgramLead(List<String> extractFields);
|
24
|
+
|
25
|
+
Iterable<ObjectNode> getCampaign();
|
26
|
+
|
27
|
+
Iterable<ObjectNode> getPrograms();
|
28
|
+
|
29
|
+
Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue);
|
30
|
+
|
31
|
+
Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues);
|
32
|
+
|
33
|
+
Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
|
34
|
+
|
35
|
+
List<MarketoField> describeCustomObject(String customObjectAPIName);
|
36
|
+
|
37
|
+
Iterable<ObjectNode> getActivityTypes();
|
38
|
+
}
|
@@ -0,0 +1,245 @@
|
|
1
|
+
package org.embulk.input.marketo;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
|
+
import com.google.common.base.Function;
|
5
|
+
import com.google.common.collect.Iterables;
|
6
|
+
import org.apache.commons.lang3.StringUtils;
|
7
|
+
import org.embulk.input.marketo.model.BulkExtractRangeHeader;
|
8
|
+
import org.embulk.input.marketo.model.MarketoField;
|
9
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
10
|
+
import org.embulk.input.marketo.rest.RecordPagingIterable;
|
11
|
+
import org.embulk.spi.DataException;
|
12
|
+
import org.embulk.spi.Exec;
|
13
|
+
import org.slf4j.Logger;
|
14
|
+
|
15
|
+
import java.io.File;
|
16
|
+
import java.io.FileOutputStream;
|
17
|
+
import java.io.IOException;
|
18
|
+
import java.io.InputStream;
|
19
|
+
import java.io.OutputStream;
|
20
|
+
import java.util.Date;
|
21
|
+
import java.util.List;
|
22
|
+
|
23
|
+
/**
|
24
|
+
* Created by tai.khuu on 9/6/17.
|
25
|
+
*/
|
26
|
+
public class MarketoServiceImpl implements MarketoService
|
27
|
+
{
|
28
|
+
private static final Logger LOGGER = Exec.getLogger(MarketoServiceImpl.class);
|
29
|
+
|
30
|
+
private static final String DEFAULT_FILE_FORMAT = "csv";
|
31
|
+
|
32
|
+
private static final int BUF_SIZE = 0x1000;
|
33
|
+
|
34
|
+
private static final int MAX_RESUME_TIME = 50;
|
35
|
+
|
36
|
+
private MarketoRestClient marketoRestClient;
|
37
|
+
|
38
|
+
public MarketoServiceImpl(MarketoRestClient marketoRestClient)
|
39
|
+
{
|
40
|
+
this.marketoRestClient = marketoRestClient;
|
41
|
+
}
|
42
|
+
|
43
|
+
@Override
|
44
|
+
public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
|
45
|
+
{
|
46
|
+
final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
|
47
|
+
marketoRestClient.startLeadBulkExtract(exportID);
|
48
|
+
try {
|
49
|
+
marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
|
50
|
+
}
|
51
|
+
catch (InterruptedException e) {
|
52
|
+
LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
|
53
|
+
throw new DataException("Error when wait for bulk extract");
|
54
|
+
}
|
55
|
+
return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
|
56
|
+
{
|
57
|
+
@Override
|
58
|
+
public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
|
59
|
+
{
|
60
|
+
return marketoRestClient.getLeadBulkExtractResult(exportID, bulkExtractRangeHeader);
|
61
|
+
}
|
62
|
+
});
|
63
|
+
}
|
64
|
+
|
65
|
+
private long saveExtractedFile(InputStream extractResult, File tempFile) throws DownloadBulkExtractException
|
66
|
+
{
|
67
|
+
long total = 0;
|
68
|
+
try (OutputStream fileOuputStream = new FileOutputStream(tempFile, true)) {
|
69
|
+
byte[] buf = new byte[BUF_SIZE];
|
70
|
+
while (true) {
|
71
|
+
int r = extractResult.read(buf);
|
72
|
+
if (r == -1) {
|
73
|
+
break;
|
74
|
+
}
|
75
|
+
fileOuputStream.write(buf, 0, r);
|
76
|
+
total += r;
|
77
|
+
}
|
78
|
+
}
|
79
|
+
catch (IOException e) {
|
80
|
+
LOGGER.error("Encounter exception when download bulk extract file", e);
|
81
|
+
throw new DownloadBulkExtractException("Encounter exception when download bulk extract file", e, total);
|
82
|
+
}
|
83
|
+
return total;
|
84
|
+
}
|
85
|
+
|
86
|
+
@Override
|
87
|
+
public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
|
88
|
+
{
|
89
|
+
final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
|
90
|
+
marketoRestClient.startActitvityBulkExtract(exportID);
|
91
|
+
try {
|
92
|
+
marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
|
93
|
+
}
|
94
|
+
catch (InterruptedException e) {
|
95
|
+
LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
|
96
|
+
throw new DataException("Error when wait for bulk extract");
|
97
|
+
}
|
98
|
+
return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
|
99
|
+
{
|
100
|
+
@Override
|
101
|
+
public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
|
102
|
+
{
|
103
|
+
return marketoRestClient.getActivitiesBulkExtractResult(exportID, bulkExtractRangeHeader);
|
104
|
+
}
|
105
|
+
});
|
106
|
+
}
|
107
|
+
private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
|
108
|
+
{
|
109
|
+
final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
|
110
|
+
long startByte = 0;
|
111
|
+
int resumeTime = 0;
|
112
|
+
while (resumeTime < MAX_RESUME_TIME) {
|
113
|
+
BulkExtractRangeHeader bulkExtractRangeHeader = new BulkExtractRangeHeader(startByte);
|
114
|
+
InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
|
115
|
+
try {
|
116
|
+
saveExtractedFile(bulkExtractResult, tempFile);
|
117
|
+
return tempFile;
|
118
|
+
}
|
119
|
+
catch (DownloadBulkExtractException e) {
|
120
|
+
startByte = startByte + e.getByteWritten();
|
121
|
+
LOGGER.warn("will resume activity bulk extract at byte [{}]", startByte);
|
122
|
+
}
|
123
|
+
resumeTime = resumeTime + 1;
|
124
|
+
}
|
125
|
+
//Too many resume we still can't get the file
|
126
|
+
throw new DataException("Can't down load bulk extract");
|
127
|
+
}
|
128
|
+
@Override
|
129
|
+
public Iterable<ObjectNode> getAllListLead(List<String> fieldNames)
|
130
|
+
{
|
131
|
+
RecordPagingIterable<ObjectNode> lists = marketoRestClient.getLists();
|
132
|
+
final String fieldNameString = StringUtils.join(fieldNames, ",");
|
133
|
+
return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
|
134
|
+
{
|
135
|
+
@Override
|
136
|
+
public Iterable<ObjectNode> apply(ObjectNode input)
|
137
|
+
{
|
138
|
+
final String id = input.get("id").asText();
|
139
|
+
return Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
|
140
|
+
{
|
141
|
+
@Override
|
142
|
+
public ObjectNode apply(ObjectNode input)
|
143
|
+
{
|
144
|
+
input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
|
145
|
+
return input;
|
146
|
+
}
|
147
|
+
});
|
148
|
+
}
|
149
|
+
});
|
150
|
+
}
|
151
|
+
|
152
|
+
@Override
|
153
|
+
public Iterable<ObjectNode> getAllProgramLead(List<String> fieldNames)
|
154
|
+
{
|
155
|
+
RecordPagingIterable<ObjectNode> lists = marketoRestClient.getPrograms();
|
156
|
+
final String fieldNameString = StringUtils.join(fieldNames, ",");
|
157
|
+
return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
|
158
|
+
{
|
159
|
+
@Override
|
160
|
+
public Iterable<ObjectNode> apply(ObjectNode input)
|
161
|
+
{
|
162
|
+
final String id = input.get("id").asText();
|
163
|
+
return Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
|
164
|
+
{
|
165
|
+
@Override
|
166
|
+
public ObjectNode apply(ObjectNode input)
|
167
|
+
{
|
168
|
+
input.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id);
|
169
|
+
return input;
|
170
|
+
}
|
171
|
+
});
|
172
|
+
}
|
173
|
+
});
|
174
|
+
}
|
175
|
+
|
176
|
+
@Override
|
177
|
+
public RecordPagingIterable<ObjectNode> getCampaign()
|
178
|
+
{
|
179
|
+
return marketoRestClient.getCampaign();
|
180
|
+
}
|
181
|
+
|
182
|
+
@Override
|
183
|
+
public List<MarketoField> describeLead()
|
184
|
+
{
|
185
|
+
return marketoRestClient.describeLead();
|
186
|
+
}
|
187
|
+
|
188
|
+
private static class DownloadBulkExtractException extends Exception
|
189
|
+
{
|
190
|
+
private final long byteWritten;
|
191
|
+
|
192
|
+
public DownloadBulkExtractException(String message, Throwable cause, long byteWritten)
|
193
|
+
{
|
194
|
+
super(message, cause);
|
195
|
+
this.byteWritten = byteWritten;
|
196
|
+
}
|
197
|
+
|
198
|
+
public DownloadBulkExtractException(Throwable cause, long byteWritten)
|
199
|
+
{
|
200
|
+
super(cause);
|
201
|
+
this.byteWritten = byteWritten;
|
202
|
+
}
|
203
|
+
|
204
|
+
public long getByteWritten()
|
205
|
+
{
|
206
|
+
return byteWritten;
|
207
|
+
}
|
208
|
+
}
|
209
|
+
|
210
|
+
@Override
|
211
|
+
public Iterable<ObjectNode> getPrograms()
|
212
|
+
{
|
213
|
+
return marketoRestClient.getPrograms();
|
214
|
+
}
|
215
|
+
|
216
|
+
@Override
|
217
|
+
public Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue)
|
218
|
+
{
|
219
|
+
return marketoRestClient.getProgramsByTag(tagType, tagValue);
|
220
|
+
}
|
221
|
+
|
222
|
+
@Override
|
223
|
+
public Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues)
|
224
|
+
{
|
225
|
+
return marketoRestClient.getProgramsByDateRange(earliestUpdatedAt, latestUpdatedAt, filterType, filterValues);
|
226
|
+
}
|
227
|
+
|
228
|
+
@Override
|
229
|
+
public List<MarketoField> describeCustomObject(String customObjectAPIName)
|
230
|
+
{
|
231
|
+
return marketoRestClient.describeCustomObject(customObjectAPIName);
|
232
|
+
}
|
233
|
+
|
234
|
+
@Override
|
235
|
+
public Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue)
|
236
|
+
{
|
237
|
+
return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
|
238
|
+
}
|
239
|
+
|
240
|
+
@Override
|
241
|
+
public Iterable<ObjectNode> getActivityTypes()
|
242
|
+
{
|
243
|
+
return marketoRestClient.getActivityTypes();
|
244
|
+
}
|
245
|
+
}
|
@@ -0,0 +1,212 @@
|
|
1
|
+
package org.embulk.input.marketo;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
4
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
5
|
+
import com.google.common.base.Function;
|
6
|
+
import com.google.common.collect.Sets;
|
7
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
8
|
+
import org.embulk.base.restclient.jackson.JacksonServiceRecord;
|
9
|
+
import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
|
10
|
+
import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
|
11
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
12
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
13
|
+
import org.embulk.input.marketo.model.MarketoField;
|
14
|
+
import org.embulk.spi.Exec;
|
15
|
+
import org.embulk.spi.util.RetryExecutor;
|
16
|
+
import org.joda.time.DateTime;
|
17
|
+
import org.slf4j.Logger;
|
18
|
+
|
19
|
+
import javax.annotation.Nullable;
|
20
|
+
|
21
|
+
import java.util.ArrayList;
|
22
|
+
import java.util.Iterator;
|
23
|
+
import java.util.List;
|
24
|
+
import java.util.NoSuchElementException;
|
25
|
+
import java.util.Set;
|
26
|
+
|
27
|
+
/**
|
28
|
+
* Created by tai.khuu on 9/18/17.
|
29
|
+
*/
|
30
|
+
public class MarketoUtils
|
31
|
+
{
|
32
|
+
public static final String MARKETO_DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
|
33
|
+
public static final String MARKETO_DATE_FORMAT = "%Y-%m-%d";
|
34
|
+
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
35
|
+
public static final Function<ObjectNode, ServiceRecord> TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION = new Function<ObjectNode, ServiceRecord>()
|
36
|
+
{
|
37
|
+
@Nullable
|
38
|
+
@Override
|
39
|
+
public JacksonServiceRecord apply(@Nullable ObjectNode input)
|
40
|
+
{
|
41
|
+
return new JacksonServiceRecord(input);
|
42
|
+
}
|
43
|
+
};
|
44
|
+
|
45
|
+
public static final String MARKETO_DATE_SIMPLE_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
|
46
|
+
|
47
|
+
public static final String LIST_ID_COLUMN_NAME = "listId";
|
48
|
+
|
49
|
+
public static final String PROGRAM_ID_COLUMN_NAME = "programId";
|
50
|
+
|
51
|
+
private MarketoUtils()
|
52
|
+
{
|
53
|
+
}
|
54
|
+
|
55
|
+
public static ServiceResponseMapper<? extends ValueLocator> buildDynamicResponseMapper(String prefix, List<MarketoField> columns)
|
56
|
+
{
|
57
|
+
JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
|
58
|
+
for (MarketoField column : columns) {
|
59
|
+
String columName = buildColumnName(prefix, column.getName());
|
60
|
+
MarketoField.MarketoDataType marketoDataType = column.getMarketoDataType();
|
61
|
+
if (marketoDataType.getFormat().isPresent()) {
|
62
|
+
builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType(), marketoDataType.getFormat().get());
|
63
|
+
}
|
64
|
+
else {
|
65
|
+
builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType());
|
66
|
+
}
|
67
|
+
}
|
68
|
+
return builder.build();
|
69
|
+
}
|
70
|
+
|
71
|
+
public static List<String> getFieldNameFromMarketoFields(List<MarketoField> columns, String... excludedFields)
|
72
|
+
{
|
73
|
+
Set<String> excludeFields = Sets.newHashSet(excludedFields);
|
74
|
+
List<String> extractedFields = new ArrayList<>();
|
75
|
+
for (MarketoField column : columns) {
|
76
|
+
if (excludeFields.contains(column.getName())) {
|
77
|
+
continue;
|
78
|
+
}
|
79
|
+
extractedFields.add(column.getName());
|
80
|
+
}
|
81
|
+
return extractedFields;
|
82
|
+
}
|
83
|
+
|
84
|
+
public static String buildColumnName(String prefix, String columnName)
|
85
|
+
{
|
86
|
+
return prefix + "_" + columnName;
|
87
|
+
}
|
88
|
+
|
89
|
+
public static final List<DateRange> sliceRange(DateTime fromDate, DateTime toDate, int rangeSize)
|
90
|
+
{
|
91
|
+
List<DateRange> ranges = new ArrayList<>();
|
92
|
+
while (fromDate.isBefore(toDate)) {
|
93
|
+
DateTime nextToDate = fromDate.plusDays(rangeSize);
|
94
|
+
if (nextToDate.isAfter(toDate)) {
|
95
|
+
ranges.add(new DateRange(fromDate, toDate));
|
96
|
+
break;
|
97
|
+
}
|
98
|
+
ranges.add(new DateRange(fromDate, nextToDate));
|
99
|
+
fromDate = nextToDate.plusSeconds(1);
|
100
|
+
}
|
101
|
+
return ranges;
|
102
|
+
}
|
103
|
+
|
104
|
+
public static String getIdentityEndPoint(String accountId)
|
105
|
+
{
|
106
|
+
return "https://" + accountId + ".mktorest.com/identity";
|
107
|
+
}
|
108
|
+
|
109
|
+
public static String getEndPoint(String accountID)
|
110
|
+
{
|
111
|
+
return "https://" + accountID + ".mktorest.com";
|
112
|
+
}
|
113
|
+
|
114
|
+
public static final class DateRange
|
115
|
+
{
|
116
|
+
public final DateTime fromDate;
|
117
|
+
public final DateTime toDate;
|
118
|
+
|
119
|
+
public DateRange(DateTime fromDate, DateTime toDate)
|
120
|
+
{
|
121
|
+
this.fromDate = fromDate;
|
122
|
+
this.toDate = toDate;
|
123
|
+
}
|
124
|
+
|
125
|
+
@Override
|
126
|
+
public String toString()
|
127
|
+
{
|
128
|
+
return "DateRange{" +
|
129
|
+
"fromDate=" + fromDate +
|
130
|
+
", toDate=" + toDate +
|
131
|
+
'}';
|
132
|
+
}
|
133
|
+
}
|
134
|
+
|
135
|
+
public static <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable) throws RetryExecutor.RetryGiveupException, InterruptedException
|
136
|
+
{
|
137
|
+
return RetryExecutor
|
138
|
+
.retryExecutor()
|
139
|
+
.withRetryLimit(maximumRetries)
|
140
|
+
.withInitialRetryWait(initialRetryIntervalMillis)
|
141
|
+
.withMaxRetryWait(maximumRetryIntervalMillis)
|
142
|
+
.runInterruptible(alwaysRetryRetryable);
|
143
|
+
}
|
144
|
+
|
145
|
+
public abstract static class AlwaysRetryRetryable<T> implements RetryExecutor.Retryable<T>
|
146
|
+
{
|
147
|
+
private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
|
148
|
+
|
149
|
+
@Override
|
150
|
+
public abstract T call() throws Exception;
|
151
|
+
|
152
|
+
@Override
|
153
|
+
public boolean isRetryableException(Exception exception)
|
154
|
+
{
|
155
|
+
return true;
|
156
|
+
}
|
157
|
+
|
158
|
+
@Override
|
159
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
|
160
|
+
{
|
161
|
+
LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
|
162
|
+
}
|
163
|
+
|
164
|
+
@Override
|
165
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
|
166
|
+
{
|
167
|
+
LOGGER.info("Giving up execution on exception", lastException);
|
168
|
+
}
|
169
|
+
}
|
170
|
+
public static <T, R> Iterable<R> flatMap(final Iterable<T> iterable, final Function<T, Iterable<R>> function)
|
171
|
+
{
|
172
|
+
final Iterator<T> iterator = iterable.iterator();
|
173
|
+
return new Iterable<R>()
|
174
|
+
{
|
175
|
+
@Override
|
176
|
+
public Iterator<R> iterator()
|
177
|
+
{
|
178
|
+
return new Iterator<R>()
|
179
|
+
{
|
180
|
+
Iterator<R> currentIterator;
|
181
|
+
@Override
|
182
|
+
public boolean hasNext()
|
183
|
+
{
|
184
|
+
if (currentIterator != null && currentIterator.hasNext()) {
|
185
|
+
return true;
|
186
|
+
}
|
187
|
+
while (iterator.hasNext()) {
|
188
|
+
currentIterator = function.apply(iterator.next()).iterator();
|
189
|
+
if (currentIterator.hasNext()) {
|
190
|
+
return true;
|
191
|
+
}
|
192
|
+
}
|
193
|
+
return false;
|
194
|
+
}
|
195
|
+
@Override
|
196
|
+
public R next()
|
197
|
+
{
|
198
|
+
if (hasNext()) {
|
199
|
+
return currentIterator.next();
|
200
|
+
}
|
201
|
+
throw new NoSuchElementException();
|
202
|
+
}
|
203
|
+
@Override
|
204
|
+
public void remove()
|
205
|
+
{
|
206
|
+
throw new UnsupportedOperationException();
|
207
|
+
}
|
208
|
+
};
|
209
|
+
}
|
210
|
+
};
|
211
|
+
}
|
212
|
+
}
|