embulk-input-marketo 0.5.7.alpha.5 → 0.5.7.alpha.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/classpath/{embulk-input-marketo-0.5.7.alpha.5.jar → embulk-input-marketo-0.5.7.alpha.6.jar} +0 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +3 -1
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +1 -1
- data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +1 -2
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +64 -57
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +0 -3
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c729920ccd16968b191ec5c12cf8f982f925f80
|
4
|
+
data.tar.gz: 1cf0acc73b4c5e90b8a11fd78770ee56791dc565
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3f14afd21ae9c968c6c0e6f1ebaf324421ae4ff10c1ceb74020ea458e970f039c2b81e9cf026c40e73089986155ea116ee1d3d5cdb47b1455b529f7de5c51506
|
7
|
+
data.tar.gz: f54d8dff5fc370b8b6f4b2f793c3f90f9e9f57a510383edc83b4c996dc255951d8d6ee882b65d9146e609180a861cea793a1c2c8167d2905f44c8c479c0458c1
|
data/build.gradle
CHANGED
data/classpath/{embulk-input-marketo-0.5.7.alpha.5.jar → embulk-input-marketo-0.5.7.alpha.6.jar}
RENAMED
Binary file
|
@@ -42,8 +42,10 @@ public class MarketoInputPluginDelegate
|
|
42
42
|
@ConfigDefault("120000")
|
43
43
|
Integer getMaximumRetriesIntervalMilis();
|
44
44
|
|
45
|
+
//We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
|
46
|
+
//We will hide the real from_date, and set it when validating task
|
45
47
|
@Config("hidden_from_date")
|
46
|
-
@ConfigDefault("\"
|
48
|
+
@ConfigDefault("\"1970-01-01\"")
|
47
49
|
@Override
|
48
50
|
Date getFromDate();
|
49
51
|
|
@@ -38,7 +38,7 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
|
|
38
38
|
try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
|
39
39
|
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
40
40
|
Date fromDate = task.getFromDate();
|
41
|
-
return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().
|
41
|
+
return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
|
42
42
|
}
|
43
43
|
catch (FileNotFoundException e) {
|
44
44
|
LOGGER.error("Exception when trying to extract activity", e);
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.input.marketo.delegate;
|
2
2
|
|
3
|
-
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
3
|
import org.embulk.base.restclient.ServiceResponseMapper;
|
5
4
|
import org.embulk.base.restclient.record.ValueLocator;
|
6
5
|
import org.embulk.input.marketo.MarketoService;
|
@@ -43,7 +42,7 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
|
|
43
42
|
MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
|
44
43
|
List<String> fieldNames = task.getExtractedFields();
|
45
44
|
Date fromDate = task.getFromDate();
|
46
|
-
File file = marketoService.extractLead(fromDate, task.getToDate().
|
45
|
+
File file = marketoService.extractLead(fromDate, task.getToDate().orNull(), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
|
47
46
|
return new FileInputStream(file);
|
48
47
|
}
|
49
48
|
catch (FileNotFoundException e) {
|
@@ -80,6 +80,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
80
80
|
@ConfigDefault("3600")
|
81
81
|
Integer getBulkJobTimeoutSecond();
|
82
82
|
|
83
|
+
@Config("incremental")
|
84
|
+
@ConfigDefault("true")
|
85
|
+
Boolean getIncremental();
|
86
|
+
|
83
87
|
@Config("latest_uids")
|
84
88
|
@ConfigDefault("[]")
|
85
89
|
Set<String> getPreviousUids();
|
@@ -134,7 +138,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
134
138
|
ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
|
135
139
|
Long currentLatestFetchTime = 0L;
|
136
140
|
Set latestUIds = null;
|
137
|
-
if (incrementalColumn != null) {
|
141
|
+
if (incrementalColumn != null && task.getIncremental()) {
|
138
142
|
int imported = 0;
|
139
143
|
DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
|
140
144
|
for (TaskReport taskReport : taskReports) {
|
@@ -149,7 +153,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
149
153
|
imported = imported + taskReport.get(Integer.class, IMPORTED_RECORD_COUNT);
|
150
154
|
}
|
151
155
|
// in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
|
152
|
-
configDiff.set(FROM_DATE, df.format(task.getToDate().
|
156
|
+
configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
|
153
157
|
configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
|
154
158
|
configDiff.set(LATEST_UID_LIST, latestUIds);
|
155
159
|
}
|
@@ -178,46 +182,46 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
178
182
|
{
|
179
183
|
final JsonParser jsonParser = new JsonParser();
|
180
184
|
Schema schema = pageBuilder.getSchema();
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
public void booleanColumn(Column column)
|
185
|
+
for (int i = 1; i <= PREVIEW_RECORD_LIMIT; i++) {
|
186
|
+
final int rowNum = i;
|
187
|
+
schema.visitColumns(new ColumnVisitor()
|
185
188
|
{
|
186
|
-
|
187
|
-
|
189
|
+
@Override
|
190
|
+
public void booleanColumn(Column column)
|
191
|
+
{
|
192
|
+
pageBuilder.setBoolean(column, false);
|
193
|
+
}
|
188
194
|
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
195
|
+
@Override
|
196
|
+
public void longColumn(Column column)
|
197
|
+
{
|
198
|
+
pageBuilder.setLong(column, 12345L);
|
199
|
+
}
|
194
200
|
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
201
|
+
@Override
|
202
|
+
public void doubleColumn(Column column)
|
203
|
+
{
|
204
|
+
pageBuilder.setDouble(column, 12345.123);
|
205
|
+
}
|
200
206
|
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
207
|
+
@Override
|
208
|
+
public void stringColumn(Column column)
|
209
|
+
{
|
210
|
+
pageBuilder.setString(column, column.getName() + "_" + rowNum);
|
211
|
+
}
|
206
212
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
213
|
+
@Override
|
214
|
+
public void timestampColumn(Column column)
|
215
|
+
{
|
216
|
+
pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(System.currentTimeMillis()));
|
217
|
+
}
|
212
218
|
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
for (int i = 0; i < PREVIEW_RECORD_LIMIT; i++) {
|
220
|
-
schema.visitColumns(visitor);
|
219
|
+
@Override
|
220
|
+
public void jsonColumn(Column column)
|
221
|
+
{
|
222
|
+
pageBuilder.setJson(column, jsonParser.parse("{\"mockKey\":\"mockValue\"}"));
|
223
|
+
}
|
224
|
+
});
|
221
225
|
pageBuilder.addRecord();
|
222
226
|
}
|
223
227
|
return Exec.newTaskReport();
|
@@ -250,33 +254,36 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
250
254
|
}
|
251
255
|
}
|
252
256
|
catch (CsvTokenizer.InvalidValueException ex) {
|
253
|
-
throw new DataException("Encounter exception when parse csv file. Please check
|
257
|
+
throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
|
258
|
+
"quote or escape character.", ex);
|
254
259
|
}
|
255
260
|
final Map<String, String> kvMap = MarketoUtils.zip(headers, values);
|
256
261
|
ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(kvMap);
|
257
262
|
|
258
|
-
if (
|
259
|
-
|
260
|
-
|
261
|
-
if (uidColumn != null) {
|
262
|
-
String uid = kvMap.get(uidColumn);
|
263
|
-
if (latestUids.contains(uid)) {
|
264
|
-
//Duplicate value
|
265
|
-
continue;
|
263
|
+
if (task.getIncremental()) {
|
264
|
+
if (!kvMap.containsKey(incrementalColumn)) {
|
265
|
+
throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
|
266
266
|
}
|
267
|
-
}
|
268
|
-
String incrementalTimeStamp = kvMap.get(incrementalColumn);
|
269
|
-
long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
|
270
|
-
if (currentTimestamp < timestamp) {
|
271
|
-
currentTimestamp = timestamp;
|
272
|
-
//switch timestamp
|
273
|
-
latestUids.clear();
|
274
|
-
}
|
275
|
-
else if (currentTimestamp == timestamp) {
|
276
|
-
//timestamp is equal
|
277
267
|
if (uidColumn != null) {
|
278
|
-
|
279
|
-
latestUids.
|
268
|
+
String uid = kvMap.get(uidColumn);
|
269
|
+
if (latestUids.contains(uid)) {
|
270
|
+
//Duplicate value
|
271
|
+
continue;
|
272
|
+
}
|
273
|
+
}
|
274
|
+
String incrementalTimeStamp = kvMap.get(incrementalColumn);
|
275
|
+
long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
|
276
|
+
if (currentTimestamp < timestamp) {
|
277
|
+
currentTimestamp = timestamp;
|
278
|
+
//switch timestamp
|
279
|
+
latestUids.clear();
|
280
|
+
}
|
281
|
+
else if (currentTimestamp == timestamp) {
|
282
|
+
//timestamp is equal
|
283
|
+
if (uidColumn != null) {
|
284
|
+
JsonNode uidField = objectNode.get(uidColumn);
|
285
|
+
latestUids.add(uidField.asText());
|
286
|
+
}
|
280
287
|
}
|
281
288
|
}
|
282
289
|
recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
|
@@ -2,10 +2,7 @@ package org.embulk.input.marketo.rest;
|
|
2
2
|
|
3
3
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
4
4
|
import com.google.common.annotations.VisibleForTesting;
|
5
|
-
import com.google.common.base.Joiner;
|
6
5
|
import com.google.common.collect.ArrayListMultimap;
|
7
|
-
import com.google.common.collect.ImmutableListMultimap;
|
8
|
-
import com.google.common.collect.Maps;
|
9
6
|
import com.google.common.collect.Multimap;
|
10
7
|
import org.eclipse.jetty.client.HttpClient;
|
11
8
|
import org.eclipse.jetty.client.api.ContentProvider;
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.7.alpha.
|
4
|
+
version: 0.5.7.alpha.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tai Khuu
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09-
|
11
|
+
date: 2017-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,11 +92,11 @@ files:
|
|
92
92
|
- src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
|
93
93
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
94
94
|
- classpath/embulk-base-restclient-0.5.3.jar
|
95
|
-
- classpath/embulk-input-marketo-0.5.7.alpha.5.jar
|
96
95
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
97
96
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
98
97
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
99
98
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
99
|
+
- classpath/embulk-input-marketo-0.5.7.alpha.6.jar
|
100
100
|
homepage:
|
101
101
|
licenses:
|
102
102
|
- MIT
|