embulk-input-marketo-extended 0.6.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
- data/.gitignore +14 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +170 -0
- data/LICENSE.txt +21 -0
- data/README.md +213 -0
- data/build.gradle +103 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +169 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/marketo.rb +3 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
- data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
- data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
- data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
- data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
- data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
- data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
- data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
- data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
- data/src/test/resources/config/custom_object_config.yaml +8 -0
- data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
- data/src/test/resources/config/rest_config.yaml +3 -0
- data/src/test/resources/fixtures/activity_extract1.csv +35 -0
- data/src/test/resources/fixtures/activity_extract2.csv +22 -0
- data/src/test/resources/fixtures/activity_types.json +22 -0
- data/src/test/resources/fixtures/all_program_full.json +53 -0
- data/src/test/resources/fixtures/campaign_response.json +38 -0
- data/src/test/resources/fixtures/campaign_response_full.json +102 -0
- data/src/test/resources/fixtures/custom_object_describe.json +124 -0
- data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
- data/src/test/resources/fixtures/custom_object_expected.json +66 -0
- data/src/test/resources/fixtures/custom_object_response.json +24 -0
- data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
- data/src/test/resources/fixtures/lead_by_list.json +33 -0
- data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
- data/src/test/resources/fixtures/lead_describe.json +221 -0
- data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
- data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
- data/src/test/resources/fixtures/lead_extract1.csv +11 -0
- data/src/test/resources/fixtures/lead_response_full.json +2402 -0
- data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
- data/src/test/resources/fixtures/leads_extract2.csv +10 -0
- data/src/test/resources/fixtures/list_reponse_full.json +191 -0
- data/src/test/resources/fixtures/lists_response.json +31 -0
- data/src/test/resources/fixtures/program_response.json +71 -0
- metadata +171 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.collect.FluentIterable;
|
4
|
+
|
5
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
6
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
7
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
8
|
+
import org.embulk.input.marketo.MarketoService;
|
9
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
10
|
+
import org.embulk.input.marketo.MarketoUtils;
|
11
|
+
import org.embulk.input.marketo.model.MarketoField;
|
12
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
13
|
+
|
14
|
+
import java.util.Iterator;
|
15
|
+
import java.util.List;
|
16
|
+
|
17
|
+
/**
|
18
|
+
* Created by tai.khuu on 9/18/17.
|
19
|
+
*/
|
20
|
+
public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithListInputPlugin.PluginTask>
|
21
|
+
{
|
22
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
|
23
|
+
{
|
24
|
+
}
|
25
|
+
|
26
|
+
public LeadWithListInputPlugin()
|
27
|
+
{
|
28
|
+
}
|
29
|
+
|
30
|
+
@Override
|
31
|
+
protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
|
32
|
+
{
|
33
|
+
List<String> extractedFields = task.getExtractedFields();
|
34
|
+
// Remove LIST_ID_COLUMN_NAME when sent fields to Marketo since LIST_ID_COLUMN_NAME are added by plugin code
|
35
|
+
extractedFields.remove(MarketoUtils.LIST_ID_COLUMN_NAME);
|
36
|
+
return FluentIterable.from(marketoService.getAllListLead(extractedFields)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
|
37
|
+
}
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
41
|
+
{
|
42
|
+
try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
|
43
|
+
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
44
|
+
LeadWithListServiceResponseMapper serviceResponseMapper = new LeadWithListServiceResponseMapper(task, marketoService);
|
45
|
+
return serviceResponseMapper.buildServiceResponseMapper(task);
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
private static class LeadWithListServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
|
50
|
+
{
|
51
|
+
public LeadWithListServiceResponseMapper(LeadWithListInputPlugin.PluginTask task, MarketoService marketoService)
|
52
|
+
{
|
53
|
+
super(task, marketoService);
|
54
|
+
}
|
55
|
+
|
56
|
+
@Override
|
57
|
+
protected List<MarketoField> getLeadColumns()
|
58
|
+
{
|
59
|
+
List<MarketoField> leadColumns = super.getLeadColumns();
|
60
|
+
leadColumns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
|
61
|
+
return leadColumns;
|
62
|
+
}
|
63
|
+
}
|
64
|
+
}
|
@@ -0,0 +1,60 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.google.common.collect.FluentIterable;
|
4
|
+
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
6
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
7
|
+
import org.embulk.input.marketo.MarketoService;
|
8
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
9
|
+
import org.embulk.input.marketo.MarketoUtils;
|
10
|
+
import org.embulk.input.marketo.model.MarketoField;
|
11
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
12
|
+
|
13
|
+
import java.util.Iterator;
|
14
|
+
import java.util.List;
|
15
|
+
|
16
|
+
/**
|
17
|
+
* Created by tai.khuu on 9/18/17.
|
18
|
+
*/
|
19
|
+
public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithProgramInputPlugin.PluginTask>
|
20
|
+
{
|
21
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
|
22
|
+
{
|
23
|
+
}
|
24
|
+
|
25
|
+
@Override
|
26
|
+
protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
|
27
|
+
{
|
28
|
+
List<String> fieldNames = task.getExtractedFields();
|
29
|
+
// Remove PROGRAM_ID_COLUMN_NAME when sent fields to Marketo since PROGRAM_ID_COLUMN_NAME are added by plugin code
|
30
|
+
fieldNames.remove(MarketoUtils.PROGRAM_ID_COLUMN_NAME);
|
31
|
+
return FluentIterable.from(marketoService.getAllProgramLead(fieldNames)).
|
32
|
+
transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
|
33
|
+
}
|
34
|
+
|
35
|
+
@Override
|
36
|
+
public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
|
37
|
+
{
|
38
|
+
try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
|
39
|
+
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
40
|
+
LeadWithProgramServiceResponseMapper serviceResponseMapper = new LeadWithProgramServiceResponseMapper(task, marketoService);
|
41
|
+
return serviceResponseMapper.buildServiceResponseMapper(task);
|
42
|
+
}
|
43
|
+
}
|
44
|
+
|
45
|
+
private static class LeadWithProgramServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
|
46
|
+
{
|
47
|
+
public LeadWithProgramServiceResponseMapper(LeadWithProgramInputPlugin.PluginTask task, MarketoService marketoService)
|
48
|
+
{
|
49
|
+
super(task, marketoService);
|
50
|
+
}
|
51
|
+
|
52
|
+
@Override
|
53
|
+
protected List<MarketoField> getLeadColumns()
|
54
|
+
{
|
55
|
+
List<MarketoField> leadColumns = super.getLeadColumns();
|
56
|
+
leadColumns.add(new MarketoField(MarketoUtils.PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
|
57
|
+
return leadColumns;
|
58
|
+
}
|
59
|
+
}
|
60
|
+
}
|
@@ -0,0 +1,441 @@
|
|
1
|
+
package org.embulk.input.marketo.delegate;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
|
+
import com.google.common.base.Function;
|
5
|
+
import com.google.common.base.Optional;
|
6
|
+
import com.google.common.collect.Iterators;
|
7
|
+
import org.embulk.base.restclient.jackson.JacksonServiceRecord;
|
8
|
+
import org.embulk.base.restclient.jackson.JacksonServiceValue;
|
9
|
+
import org.embulk.base.restclient.record.RecordImporter;
|
10
|
+
import org.embulk.base.restclient.record.ServiceRecord;
|
11
|
+
import org.embulk.base.restclient.record.ValueLocator;
|
12
|
+
import org.embulk.config.Config;
|
13
|
+
import org.embulk.config.ConfigDefault;
|
14
|
+
import org.embulk.config.ConfigDiff;
|
15
|
+
import org.embulk.config.ConfigException;
|
16
|
+
import org.embulk.config.ConfigInject;
|
17
|
+
import org.embulk.config.TaskReport;
|
18
|
+
import org.embulk.input.marketo.CsvTokenizer;
|
19
|
+
import org.embulk.input.marketo.MarketoService;
|
20
|
+
import org.embulk.input.marketo.MarketoServiceImpl;
|
21
|
+
import org.embulk.input.marketo.MarketoUtils;
|
22
|
+
import org.embulk.input.marketo.rest.MarketoRestClient;
|
23
|
+
import org.embulk.spi.BufferAllocator;
|
24
|
+
import org.embulk.spi.Column;
|
25
|
+
import org.embulk.spi.ColumnVisitor;
|
26
|
+
import org.embulk.spi.DataException;
|
27
|
+
import org.embulk.spi.Exec;
|
28
|
+
import org.embulk.spi.PageBuilder;
|
29
|
+
import org.embulk.spi.Schema;
|
30
|
+
import org.embulk.spi.json.JsonParser;
|
31
|
+
import org.embulk.spi.time.Timestamp;
|
32
|
+
import org.embulk.spi.time.TimestampParser;
|
33
|
+
import org.embulk.spi.util.InputStreamFileInput;
|
34
|
+
import org.embulk.spi.util.LineDecoder;
|
35
|
+
import org.joda.time.DateTime;
|
36
|
+
import org.msgpack.value.Value;
|
37
|
+
|
38
|
+
import java.io.InputStream;
|
39
|
+
import java.text.DateFormat;
|
40
|
+
import java.text.SimpleDateFormat;
|
41
|
+
import java.util.ArrayList;
|
42
|
+
import java.util.Date;
|
43
|
+
import java.util.HashMap;
|
44
|
+
import java.util.Iterator;
|
45
|
+
import java.util.List;
|
46
|
+
import java.util.Map;
|
47
|
+
import java.util.NoSuchElementException;
|
48
|
+
|
49
|
+
/**
|
50
|
+
* Created by tai.khuu on 9/18/17.
|
51
|
+
*/
|
52
|
+
public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBulkExtractInputPlugin.PluginTask> extends MarketoBaseInputPluginDelegate<T>
|
53
|
+
{
|
54
|
+
private static final String FROM_DATE = "from_date";
|
55
|
+
|
56
|
+
private static final int MARKETO_MAX_RANGE_EXTRACT = 30;
|
57
|
+
|
58
|
+
public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
|
59
|
+
{
|
60
|
+
@Config("from_date")
|
61
|
+
Date getFromDate();
|
62
|
+
|
63
|
+
@Config("fetch_days")
|
64
|
+
@ConfigDefault("1")
|
65
|
+
Integer getFetchDays();
|
66
|
+
|
67
|
+
@Config("latest_fetch_time")
|
68
|
+
@ConfigDefault("null")
|
69
|
+
Optional<Long> getLatestFetchTime();
|
70
|
+
|
71
|
+
@ConfigInject
|
72
|
+
BufferAllocator getBufferAllocator();
|
73
|
+
|
74
|
+
@Config("polling_interval_second")
|
75
|
+
@ConfigDefault("60")
|
76
|
+
Integer getPollingIntervalSecond();
|
77
|
+
|
78
|
+
@Config("bulk_job_timeout_second")
|
79
|
+
@ConfigDefault("3600")
|
80
|
+
Integer getBulkJobTimeoutSecond();
|
81
|
+
|
82
|
+
@Config("to_date")
|
83
|
+
@ConfigDefault("null")
|
84
|
+
Optional<Date> getToDate();
|
85
|
+
|
86
|
+
void setToDate(Optional<Date> toDate);
|
87
|
+
|
88
|
+
@Config("incremental_column")
|
89
|
+
@ConfigDefault("\"createdAt\"")
|
90
|
+
//Incremental column are only keep here since we don't want to introduce too much change to plugin
|
91
|
+
//Consider remove it in next release
|
92
|
+
Optional<String> getIncrementalColumn();
|
93
|
+
|
94
|
+
void setIncrementalColumn(Optional<String> incrementalColumn);
|
95
|
+
|
96
|
+
@Config("uid_column")
|
97
|
+
@ConfigDefault("null")
|
98
|
+
Optional<String> getUidColumn();
|
99
|
+
void setUidColumn(Optional<String> uidColumn);
|
100
|
+
}
|
101
|
+
|
102
|
+
@Override
|
103
|
+
public void validateInputTask(T task)
|
104
|
+
{
|
105
|
+
super.validateInputTask(task);
|
106
|
+
if (task.getFromDate() == null) {
|
107
|
+
throw new ConfigException("From date is required for Bulk Extract");
|
108
|
+
}
|
109
|
+
if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
|
110
|
+
throw new ConfigException("From date can't not be in future");
|
111
|
+
}
|
112
|
+
if (task.getIncremental()
|
113
|
+
&& task.getIncrementalColumn().isPresent()
|
114
|
+
&& task.getIncrementalColumn().get().equals("updatedAt")) {
|
115
|
+
throw new ConfigException("Column 'updatedAt' cannot be incremental imported");
|
116
|
+
}
|
117
|
+
//Calculate to date
|
118
|
+
DateTime toDate = getToDate(task);
|
119
|
+
task.setToDate(Optional.of(toDate.toDate()));
|
120
|
+
}
|
121
|
+
|
122
|
+
public DateTime getToDate(T task)
|
123
|
+
{
|
124
|
+
Date fromDate = task.getFromDate();
|
125
|
+
DateTime dateTime = new DateTime(fromDate);
|
126
|
+
DateTime toDate = dateTime.plusDays(task.getFetchDays());
|
127
|
+
if (toDate.isAfter(task.getJobStartTime())) {
|
128
|
+
//Lock down to date
|
129
|
+
toDate = task.getJobStartTime();
|
130
|
+
}
|
131
|
+
return toDate;
|
132
|
+
}
|
133
|
+
|
134
|
+
@Override
|
135
|
+
public ConfigDiff buildConfigDiff(T task, Schema schema, int taskCount, List<TaskReport> taskReports)
|
136
|
+
{
|
137
|
+
ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
|
138
|
+
String incrementalColumn = task.getIncrementalColumn().orNull();
|
139
|
+
if (incrementalColumn != null && task.getIncremental()) {
|
140
|
+
DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
|
141
|
+
// We will always move the range forward.
|
142
|
+
Date toDate = task.getToDate().orNull();
|
143
|
+
configDiff.set(FROM_DATE, df.format(toDate));
|
144
|
+
}
|
145
|
+
return configDiff;
|
146
|
+
}
|
147
|
+
|
148
|
+
@Override
|
149
|
+
public TaskReport ingestServiceData(final T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
|
150
|
+
{
|
151
|
+
TaskReport taskReport = Exec.newTaskReport();
|
152
|
+
if (Exec.isPreview()) {
|
153
|
+
return importMockPreviewData(pageBuilder);
|
154
|
+
}
|
155
|
+
else {
|
156
|
+
try (LineDecoderIterator decoderIterator = getLineDecoderIterator(task)) {
|
157
|
+
Iterator<Map<String, String>> csvRecords = Iterators.concat(Iterators.transform(decoderIterator, new Function<LineDecoder, Iterator<Map<String, String>>>()
|
158
|
+
{
|
159
|
+
@Override
|
160
|
+
public Iterator<Map<String, String>> apply(LineDecoder input)
|
161
|
+
{
|
162
|
+
return new CsvRecordIterator(input, task);
|
163
|
+
}
|
164
|
+
}));
|
165
|
+
//Keep the preview code here when we can enable real preview
|
166
|
+
if (Exec.isPreview()) {
|
167
|
+
csvRecords = Iterators.limit(csvRecords, PREVIEW_RECORD_LIMIT);
|
168
|
+
}
|
169
|
+
int imported = 0;
|
170
|
+
while (csvRecords.hasNext()) {
|
171
|
+
Map<String, String> csvRecord = csvRecords.next();
|
172
|
+
ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
|
173
|
+
recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
|
174
|
+
imported = imported + 1;
|
175
|
+
}
|
176
|
+
return taskReport;
|
177
|
+
}
|
178
|
+
}
|
179
|
+
}
|
180
|
+
|
181
|
+
/**
|
182
|
+
* This method should be removed when we allow skip preview phase
|
183
|
+
* @param pageBuilder
|
184
|
+
* @return TaskReport
|
185
|
+
*/
|
186
|
+
private TaskReport importMockPreviewData(final PageBuilder pageBuilder)
|
187
|
+
{
|
188
|
+
final JsonParser jsonParser = new JsonParser();
|
189
|
+
Schema schema = pageBuilder.getSchema();
|
190
|
+
for (int i = 1; i <= PREVIEW_RECORD_LIMIT; i++) {
|
191
|
+
final int rowNum = i;
|
192
|
+
schema.visitColumns(new ColumnVisitor()
|
193
|
+
{
|
194
|
+
@Override
|
195
|
+
public void booleanColumn(Column column)
|
196
|
+
{
|
197
|
+
pageBuilder.setBoolean(column, false);
|
198
|
+
}
|
199
|
+
|
200
|
+
@Override
|
201
|
+
public void longColumn(Column column)
|
202
|
+
{
|
203
|
+
pageBuilder.setLong(column, 12345L);
|
204
|
+
}
|
205
|
+
|
206
|
+
@Override
|
207
|
+
public void doubleColumn(Column column)
|
208
|
+
{
|
209
|
+
pageBuilder.setDouble(column, 12345.123);
|
210
|
+
}
|
211
|
+
|
212
|
+
@Override
|
213
|
+
public void stringColumn(Column column)
|
214
|
+
{
|
215
|
+
pageBuilder.setString(column, column.getName() + "_" + rowNum);
|
216
|
+
}
|
217
|
+
|
218
|
+
@Override
|
219
|
+
public void timestampColumn(Column column)
|
220
|
+
{
|
221
|
+
pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(System.currentTimeMillis()));
|
222
|
+
}
|
223
|
+
|
224
|
+
@Override
|
225
|
+
public void jsonColumn(Column column)
|
226
|
+
{
|
227
|
+
pageBuilder.setJson(column, jsonParser.parse("{\"mockKey\":\"mockValue\"}"));
|
228
|
+
}
|
229
|
+
});
|
230
|
+
pageBuilder.addRecord();
|
231
|
+
}
|
232
|
+
return Exec.newTaskReport();
|
233
|
+
}
|
234
|
+
|
235
|
+
private LineDecoderIterator getLineDecoderIterator(T task)
|
236
|
+
{
|
237
|
+
List<MarketoUtils.DateRange> dateRanges = MarketoUtils.sliceRange(new DateTime(task.getFromDate()), new DateTime(task.getToDate().orNull()), MARKETO_MAX_RANGE_EXTRACT);
|
238
|
+
final Iterator<MarketoUtils.DateRange> iterator = dateRanges.iterator();
|
239
|
+
return new LineDecoderIterator(iterator, task);
|
240
|
+
}
|
241
|
+
|
242
|
+
@Override
|
243
|
+
protected final Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, T task)
|
244
|
+
{
|
245
|
+
throw new UnsupportedOperationException();
|
246
|
+
}
|
247
|
+
|
248
|
+
protected abstract InputStream getExtractedStream(MarketoService service, T task, DateTime fromDate, DateTime toDate);
|
249
|
+
|
250
|
+
private static class AllStringJacksonServiceRecord extends JacksonServiceRecord
|
251
|
+
{
|
252
|
+
public AllStringJacksonServiceRecord(ObjectNode record)
|
253
|
+
{
|
254
|
+
super(record);
|
255
|
+
}
|
256
|
+
|
257
|
+
@Override
|
258
|
+
public JacksonServiceValue getValue(ValueLocator locator)
|
259
|
+
{
|
260
|
+
// We know that this thing only contain text.
|
261
|
+
JacksonServiceValue value = super.getValue(locator);
|
262
|
+
return new StringConverterJacksonServiceRecord(value.stringValue());
|
263
|
+
}
|
264
|
+
}
|
265
|
+
|
266
|
+
private static class StringConverterJacksonServiceRecord extends JacksonServiceValue
|
267
|
+
{
|
268
|
+
private String textValue;
|
269
|
+
|
270
|
+
public StringConverterJacksonServiceRecord(String textValue)
|
271
|
+
{
|
272
|
+
super(null);
|
273
|
+
this.textValue = textValue;
|
274
|
+
}
|
275
|
+
|
276
|
+
@Override
|
277
|
+
public boolean isNull()
|
278
|
+
{
|
279
|
+
return textValue == null || textValue.equals("null");
|
280
|
+
}
|
281
|
+
|
282
|
+
@Override
|
283
|
+
public boolean booleanValue()
|
284
|
+
{
|
285
|
+
return Boolean.parseBoolean(textValue);
|
286
|
+
}
|
287
|
+
|
288
|
+
@Override
|
289
|
+
public double doubleValue()
|
290
|
+
{
|
291
|
+
return Double.parseDouble(textValue);
|
292
|
+
}
|
293
|
+
|
294
|
+
@Override
|
295
|
+
public Value jsonValue(JsonParser jsonParser)
|
296
|
+
{
|
297
|
+
return jsonParser.parse(textValue);
|
298
|
+
}
|
299
|
+
|
300
|
+
@Override
|
301
|
+
public long longValue()
|
302
|
+
{
|
303
|
+
return Long.parseLong(textValue);
|
304
|
+
}
|
305
|
+
|
306
|
+
@Override
|
307
|
+
public String stringValue()
|
308
|
+
{
|
309
|
+
return textValue;
|
310
|
+
}
|
311
|
+
|
312
|
+
@Override
|
313
|
+
public Timestamp timestampValue(TimestampParser timestampParser)
|
314
|
+
{
|
315
|
+
return timestampParser.parse(textValue);
|
316
|
+
}
|
317
|
+
}
|
318
|
+
|
319
|
+
private final class LineDecoderIterator implements Iterator<LineDecoder>, AutoCloseable
|
320
|
+
{
|
321
|
+
private LineDecoder currentLineDecoder;
|
322
|
+
|
323
|
+
private Iterator<MarketoUtils.DateRange> dateRangeIterator;
|
324
|
+
|
325
|
+
private MarketoService marketoService;
|
326
|
+
|
327
|
+
private MarketoRestClient marketoRestClient;
|
328
|
+
private T task;
|
329
|
+
public LineDecoderIterator(Iterator<MarketoUtils.DateRange> dateRangeIterator, T task)
|
330
|
+
{
|
331
|
+
marketoRestClient = createMarketoRestClient(task);
|
332
|
+
marketoService = new MarketoServiceImpl(marketoRestClient);
|
333
|
+
this.dateRangeIterator = dateRangeIterator;
|
334
|
+
this.task = task;
|
335
|
+
}
|
336
|
+
|
337
|
+
@Override
|
338
|
+
public void close()
|
339
|
+
{
|
340
|
+
if (currentLineDecoder != null) {
|
341
|
+
currentLineDecoder.close();
|
342
|
+
}
|
343
|
+
if (marketoRestClient != null) {
|
344
|
+
marketoRestClient.close();
|
345
|
+
}
|
346
|
+
}
|
347
|
+
|
348
|
+
@Override
|
349
|
+
public boolean hasNext()
|
350
|
+
{
|
351
|
+
return dateRangeIterator.hasNext();
|
352
|
+
}
|
353
|
+
|
354
|
+
@Override
|
355
|
+
public LineDecoder next()
|
356
|
+
{
|
357
|
+
if (hasNext()) {
|
358
|
+
MarketoUtils.DateRange next = dateRangeIterator.next();
|
359
|
+
InputStream extractedStream = getExtractedStream(marketoService, task, next.fromDate, next.toDate);
|
360
|
+
currentLineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), extractedStream), task);
|
361
|
+
return currentLineDecoder;
|
362
|
+
}
|
363
|
+
throw new NoSuchElementException();
|
364
|
+
}
|
365
|
+
|
366
|
+
@Override
|
367
|
+
public void remove()
|
368
|
+
{
|
369
|
+
throw new UnsupportedOperationException("Removed are not supported");
|
370
|
+
}
|
371
|
+
}
|
372
|
+
|
373
|
+
private class CsvRecordIterator implements Iterator<Map<String, String>>
|
374
|
+
{
|
375
|
+
private CsvTokenizer tokenizer;
|
376
|
+
|
377
|
+
private List<String> headers;
|
378
|
+
|
379
|
+
private Map<String, String> currentCsvRecord;
|
380
|
+
public CsvRecordIterator(LineDecoder lineDecoder, T task)
|
381
|
+
{
|
382
|
+
tokenizer = new CsvTokenizer(lineDecoder, task);
|
383
|
+
if (!tokenizer.nextFile()) {
|
384
|
+
throw new DataException("Can't read extract input stream");
|
385
|
+
}
|
386
|
+
headers = new ArrayList<>();
|
387
|
+
tokenizer.nextRecord();
|
388
|
+
while (tokenizer.hasNextColumn()) {
|
389
|
+
headers.add(tokenizer.nextColumn());
|
390
|
+
}
|
391
|
+
}
|
392
|
+
|
393
|
+
@Override
|
394
|
+
public boolean hasNext()
|
395
|
+
{
|
396
|
+
if (currentCsvRecord == null) {
|
397
|
+
currentCsvRecord = getNextCSVRecord();
|
398
|
+
}
|
399
|
+
return currentCsvRecord != null;
|
400
|
+
}
|
401
|
+
|
402
|
+
@Override
|
403
|
+
public Map<String, String> next()
|
404
|
+
{
|
405
|
+
try {
|
406
|
+
if (hasNext()) {
|
407
|
+
return currentCsvRecord;
|
408
|
+
}
|
409
|
+
}
|
410
|
+
finally {
|
411
|
+
currentCsvRecord = null;
|
412
|
+
}
|
413
|
+
throw new NoSuchElementException();
|
414
|
+
}
|
415
|
+
|
416
|
+
@Override
|
417
|
+
public void remove()
|
418
|
+
{
|
419
|
+
throw new UnsupportedOperationException();
|
420
|
+
}
|
421
|
+
private Map<String, String> getNextCSVRecord()
|
422
|
+
{
|
423
|
+
if (!tokenizer.nextRecord()) {
|
424
|
+
return null;
|
425
|
+
}
|
426
|
+
Map<String, String> kvMap = new HashMap<>();
|
427
|
+
try {
|
428
|
+
int i = 0;
|
429
|
+
while (tokenizer.hasNextColumn()) {
|
430
|
+
kvMap.put(headers.get(i), tokenizer.nextColumnOrNull());
|
431
|
+
i++;
|
432
|
+
}
|
433
|
+
}
|
434
|
+
catch (CsvTokenizer.InvalidValueException ex) {
|
435
|
+
throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
|
436
|
+
"quote or escape character.", ex);
|
437
|
+
}
|
438
|
+
return kvMap;
|
439
|
+
}
|
440
|
+
}
|
441
|
+
}
|