embulk-input-marketo 0.6.8 → 0.6.9.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +0 -12
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +1 -1
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +20 -12
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +14 -10
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +11 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f075ab4c25711804d71689227cd164392154cb09
|
4
|
+
data.tar.gz: a565d6381ac3ac89b21397c6b522c4c3192c6f60
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7593bdf9bacc02e3e13fc28009c12ef68b69790022d66613fa58fd27227f7616a8a4c4e15843d75e474fc84922ae53ce1e6ae63dd8363d45553f16a09484f0c0
|
7
|
+
data.tar.gz: 7307a87215da07e29703d537603d6b49fdfaf985fb61f30998f4153ab7ce8a14d21144ffc9c0ce8fe7492edcb1e04a14b7cd5a6f4702b94a666b9ac92b045b1a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.6.8 - 2018-04-12
|
2
|
+
* [fixed] Fix incorrect incorrect retry logic [#84](https://github.com/treasure-data/embulk-input-marketo/pull/84)
|
3
|
+
|
1
4
|
## 0.6.7 - 2018-02-26
|
2
5
|
* [fixed] Remove de-duplication logic [#83](https://github.com/treasure-data/embulk-input-marketo/pull/83)
|
3
6
|
|
data/build.gradle
CHANGED
@@ -30,18 +30,6 @@ public class MarketoInputPluginDelegate
|
|
30
30
|
@Config("target")
|
31
31
|
Target getTarget();
|
32
32
|
|
33
|
-
@Config("maximum_retries")
|
34
|
-
@ConfigDefault("3")
|
35
|
-
Integer getMaximumRetries();
|
36
|
-
|
37
|
-
@Config("initial_retry_interval_milis")
|
38
|
-
@ConfigDefault("20000")
|
39
|
-
Integer getInitialRetryIntervalMilis();
|
40
|
-
|
41
|
-
@Config("maximum_retries_interval_milis")
|
42
|
-
@ConfigDefault("120000")
|
43
|
-
Integer getMaximumRetriesIntervalMilis();
|
44
|
-
|
45
33
|
//We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
|
46
34
|
//We will hide the real from_date, and set it when validating task
|
47
35
|
@Config("hidden_from_date")
|
@@ -36,7 +36,7 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
|
|
36
36
|
extends RestClientInputTaskBase, MarketoRestClient.PluginTask
|
37
37
|
{
|
38
38
|
@Config("maximum_retries")
|
39
|
-
@ConfigDefault("
|
39
|
+
@ConfigDefault("7")
|
40
40
|
Integer getMaximumRetries();
|
41
41
|
|
42
42
|
@Config("initial_retry_interval_milis")
|
@@ -33,7 +33,7 @@ import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTR
|
|
33
33
|
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
|
34
34
|
|
35
35
|
/**
|
36
|
-
* Marketo base rest client
|
36
|
+
* Marketo base rest client
|
37
37
|
* Created by tai.khuu on 9/7/17.
|
38
38
|
*/
|
39
39
|
public class MarketoBaseRestClient implements AutoCloseable
|
@@ -44,8 +44,6 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
44
44
|
|
45
45
|
private static final String AUTHORIZATION_HEADER = "Authorization";
|
46
46
|
|
47
|
-
protected static final long READ_TIMEOUT_MILLIS = 30000;
|
48
|
-
|
49
47
|
private String identityEndPoint;
|
50
48
|
|
51
49
|
private String clientId;
|
@@ -54,19 +52,22 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
54
52
|
|
55
53
|
private String accessToken;
|
56
54
|
|
57
|
-
private int
|
55
|
+
private int marketoLimitIntervalMillis;
|
58
56
|
|
59
57
|
private Jetty92RetryHelper retryHelper;
|
60
58
|
|
59
|
+
protected long readTimeoutMillis;
|
60
|
+
|
61
61
|
protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false).configure(ALLOW_UNQUOTED_CONTROL_CHARS, false);
|
62
62
|
|
63
|
-
MarketoBaseRestClient(String identityEndPoint, String clientId, String clientSecret, int
|
63
|
+
MarketoBaseRestClient(String identityEndPoint, String clientId, String clientSecret, int marketoLimitIntervalMillis, long readTimeoutMillis, Jetty92RetryHelper retryHelper)
|
64
64
|
{
|
65
65
|
this.identityEndPoint = identityEndPoint;
|
66
66
|
this.clientId = clientId;
|
67
67
|
this.clientSecret = clientSecret;
|
68
|
+
this.readTimeoutMillis = readTimeoutMillis;
|
68
69
|
this.retryHelper = retryHelper;
|
69
|
-
this.
|
70
|
+
this.marketoLimitIntervalMillis = marketoLimitIntervalMillis;
|
70
71
|
}
|
71
72
|
|
72
73
|
private void renewAccessToken()
|
@@ -93,7 +94,7 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
93
94
|
params.put("client_id", clientId);
|
94
95
|
params.put("client_secret", clientSecret);
|
95
96
|
params.put("grant_type", "client_credentials");
|
96
|
-
String response = retryHelper.requestWithRetry(new StringJetty92ResponseEntityReader(
|
97
|
+
String response = retryHelper.requestWithRetry(new StringJetty92ResponseEntityReader(readTimeoutMillis), new Jetty92SingleRequester()
|
97
98
|
{
|
98
99
|
@Override
|
99
100
|
public void requestOnce(HttpClient client, Response.Listener responseListener)
|
@@ -116,10 +117,14 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
116
117
|
@Override
|
117
118
|
protected boolean isExceptionToRetry(Exception exception)
|
118
119
|
{
|
119
|
-
if (exception instanceof
|
120
|
+
if (exception instanceof TimeoutException || exception instanceof SocketTimeoutException || exception instanceof EOFException || super.isExceptionToRetry(exception)) {
|
121
|
+
return true;
|
122
|
+
}
|
123
|
+
// unwrap
|
124
|
+
if (exception instanceof ExecutionException || (exception instanceof IOException && exception.getCause() != null)) {
|
120
125
|
return this.toRetry((Exception) exception.getCause());
|
121
126
|
}
|
122
|
-
return
|
127
|
+
return false;
|
123
128
|
}
|
124
129
|
});
|
125
130
|
|
@@ -197,7 +202,10 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
197
202
|
@Override
|
198
203
|
protected boolean isExceptionToRetry(Exception exception)
|
199
204
|
{
|
200
|
-
if (exception instanceof
|
205
|
+
if (exception instanceof EOFException || exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception)) {
|
206
|
+
return true;
|
207
|
+
}
|
208
|
+
if (exception instanceof ExecutionException || (exception instanceof IOException && exception.getCause() != null)) {
|
201
209
|
return this.toRetry((Exception) exception.getCause());
|
202
210
|
}
|
203
211
|
if (exception instanceof MarketoAPIException) {
|
@@ -212,7 +220,7 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
212
220
|
return true;
|
213
221
|
case "606":
|
214
222
|
try {
|
215
|
-
Thread.sleep(
|
223
|
+
Thread.sleep(marketoLimitIntervalMillis);
|
216
224
|
}
|
217
225
|
catch (InterruptedException e) {
|
218
226
|
LOGGER.error("Encounter exception when waiting for interval limit", e);
|
@@ -225,7 +233,7 @@ public class MarketoBaseRestClient implements AutoCloseable
|
|
225
233
|
return false;
|
226
234
|
}
|
227
235
|
}
|
228
|
-
return
|
236
|
+
return false;
|
229
237
|
}
|
230
238
|
});
|
231
239
|
}
|
@@ -104,16 +104,20 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
104
104
|
@ConfigDefault("200")
|
105
105
|
Integer getMaxReturn();
|
106
106
|
void setMaxReturn(Integer maxReturn);
|
107
|
+
|
108
|
+
@Config("read_timeout_millis")
|
109
|
+
@ConfigDefault("60000")
|
110
|
+
Long getReadTimeoutMillis();
|
107
111
|
}
|
108
112
|
|
109
113
|
public MarketoRestClient(PluginTask task, Jetty92RetryHelper retryHelper)
|
110
114
|
{
|
111
|
-
this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getMarketoLimitIntervalMilis(), retryHelper);
|
115
|
+
this(MarketoUtils.getEndPoint(task.getAccountId()), MarketoUtils.getIdentityEndPoint(task.getAccountId()), task.getClientId(), task.getClientSecret(), task.getBatchSize(), task.getMaxReturn(), task.getReadTimeoutMillis(), task.getMarketoLimitIntervalMilis(), retryHelper);
|
112
116
|
}
|
113
117
|
|
114
|
-
public MarketoRestClient(String endPoint, String identityEndPoint, String clientId, String clientSecret, Integer batchSize, Integer maxReturn, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
|
118
|
+
public MarketoRestClient(String endPoint, String identityEndPoint, String clientId, String clientSecret, Integer batchSize, Integer maxReturn, long readTimeoutMilis, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
|
115
119
|
{
|
116
|
-
super(identityEndPoint, clientId, clientSecret, marketoLimitIntervalMilis, retryHelper);
|
120
|
+
super(identityEndPoint, clientId, clientSecret, marketoLimitIntervalMilis, readTimeoutMilis, retryHelper);
|
117
121
|
this.endPoint = endPoint;
|
118
122
|
this.batchSize = batchSize;
|
119
123
|
this.maxReturn = maxReturn;
|
@@ -121,7 +125,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
121
125
|
|
122
126
|
public List<MarketoField> describeLead()
|
123
127
|
{
|
124
|
-
MarketoResponse<ObjectNode> jsonResponse = doGet(endPoint + MarketoRESTEndpoint.DESCRIBE_LEAD.getEndpoint(), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(
|
128
|
+
MarketoResponse<ObjectNode> jsonResponse = doGet(endPoint + MarketoRESTEndpoint.DESCRIBE_LEAD.getEndpoint(), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(this.readTimeoutMillis));
|
125
129
|
List<MarketoField> marketoFields = new ArrayList<>();
|
126
130
|
List<ObjectNode> fields = jsonResponse.getResult();
|
127
131
|
for (int i = 0; i < fields.size(); i++) {
|
@@ -175,7 +179,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
175
179
|
MarketoResponse<ObjectNode> marketoResponse = null;
|
176
180
|
try {
|
177
181
|
LOGGER.info("Send bulk extract request [{}]", request);
|
178
|
-
marketoResponse = doPost(endPoint + endpoint.getEndpoint(), null, null, OBJECT_MAPPER.writeValueAsString(request), new MarketoResponseJetty92EntityReader<ObjectNode>(
|
182
|
+
marketoResponse = doPost(endPoint + endpoint.getEndpoint(), null, null, OBJECT_MAPPER.writeValueAsString(request), new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
|
179
183
|
}
|
180
184
|
catch (JsonProcessingException e) {
|
181
185
|
LOGGER.error("Encounter exception when deserialize bulk extract request", e);
|
@@ -203,7 +207,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
203
207
|
{
|
204
208
|
MarketoResponse<ObjectNode> marketoResponse = doPost(endPoint + marketoRESTEndpoint.getEndpoint(
|
205
209
|
new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, null,
|
206
|
-
new MarketoResponseJetty92EntityReader<ObjectNode>(
|
210
|
+
new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
|
207
211
|
if (!marketoResponse.isSuccess()) {
|
208
212
|
MarketoError error = marketoResponse.getErrors().get(0);
|
209
213
|
throw new DataException(String.format("Can't start job for export Job id : %s, error code: %s, error message: %s", exportId, error.getCode(), error.getMessage()));
|
@@ -245,7 +249,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
245
249
|
long now = System.currentTimeMillis();
|
246
250
|
while (true) {
|
247
251
|
MarketoResponse<ObjectNode> marketoResponse = doGet(this.endPoint + marketoRESTEndpoint.getEndpoint(
|
248
|
-
new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(
|
252
|
+
new ImmutableMap.Builder<String, String>().put("export_id", exportId).build()), null, null, new MarketoResponseJetty92EntityReader<ObjectNode>(readTimeoutMillis));
|
249
253
|
if (marketoResponse.isSuccess()) {
|
250
254
|
ObjectNode objectNode = marketoResponse.getResult().get(0);
|
251
255
|
String status = objectNode.get("status").asText();
|
@@ -290,7 +294,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
290
294
|
headers.put(RANGE_HEADER, bulkExtractRangeHeader.toRangeHeaderValue());
|
291
295
|
LOGGER.info("Range header value [{}]", bulkExtractRangeHeader.toRangeHeaderValue());
|
292
296
|
}
|
293
|
-
return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), headers, null, new MarketoInputStreamResponseEntityReader(
|
297
|
+
return doGet(this.endPoint + endpoint.getEndpoint(new ImmutableMap.Builder().put("export_id", exportId).build()), headers, null, new MarketoInputStreamResponseEntityReader(readTimeoutMillis));
|
294
298
|
}
|
295
299
|
|
296
300
|
public RecordPagingIterable<ObjectNode> getLists()
|
@@ -345,7 +349,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
345
349
|
if (parameters != null) {
|
346
350
|
params.putAll(parameters);
|
347
351
|
}
|
348
|
-
MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(
|
352
|
+
MarketoResponse<T> marketoResponse = doGet(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(readTimeoutMillis, recordClass));
|
349
353
|
return new RecordPagingIterable.OffsetPage<>(marketoResponse.getResult(), offset + marketoResponse.getResult().size(), marketoResponse.getResult().size() == maxReturn);
|
350
354
|
}
|
351
355
|
});
|
@@ -386,7 +390,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
|
|
386
390
|
}
|
387
391
|
//Let do GET Disguise in POST here to overcome Marketo URI Too long error
|
388
392
|
FormContentProvider formContentProvider = new FormContentProvider(fields);
|
389
|
-
MarketoResponse<T> marketoResponse = doPost(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(
|
393
|
+
MarketoResponse<T> marketoResponse = doPost(endPoint, null, params.build(), new MarketoResponseJetty92EntityReader<>(readTimeoutMillis, recordClass), formContentProvider);
|
390
394
|
return new RecordPagingIterable.TokenPage<>(marketoResponse.getResult(), marketoResponse.getNextPageToken(), marketoResponse.getNextPageToken() != null);
|
391
395
|
}
|
392
396
|
});
|
@@ -27,6 +27,7 @@ import org.mockito.ArgumentCaptor;
|
|
27
27
|
import org.mockito.Mockito;
|
28
28
|
|
29
29
|
import java.io.EOFException;
|
30
|
+
import java.io.IOException;
|
30
31
|
import java.net.SocketTimeoutException;
|
31
32
|
import java.nio.charset.StandardCharsets;
|
32
33
|
import java.util.HashMap;
|
@@ -53,7 +54,7 @@ public class MarketoBaseRestClientTest
|
|
53
54
|
public void prepare()
|
54
55
|
{
|
55
56
|
mockJetty92 = Mockito.mock(Jetty92RetryHelper.class);
|
56
|
-
marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, mockJetty92);
|
57
|
+
marketoBaseRestClient = new MarketoBaseRestClient("identityEndPoint", "clientId", "clientSecret", MARKETO_LIMIT_INTERVAL_MILIS, 60000, mockJetty92);
|
57
58
|
}
|
58
59
|
|
59
60
|
@Test
|
@@ -95,6 +96,10 @@ public class MarketoBaseRestClientTest
|
|
95
96
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
|
96
97
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
|
97
98
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new EOFException()));
|
99
|
+
// When EOFException is wrapped in IOException it should be retried too
|
100
|
+
Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
|
101
|
+
// Retry TimeoutException when it is wrapped in IOException
|
102
|
+
Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
|
98
103
|
}
|
99
104
|
@Test
|
100
105
|
public void testGetAccessTokenWithError()
|
@@ -201,6 +206,11 @@ public class MarketoBaseRestClientTest
|
|
201
206
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new TimeoutException())));
|
202
207
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new EOFException())));
|
203
208
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new ExecutionException(new SocketTimeoutException())));
|
209
|
+
// When EOFException is wrapped in IOException it should be retried too
|
210
|
+
Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new EOFException())));
|
211
|
+
// Retry TimeoutException when it is wrapped in IOException
|
212
|
+
Assert.assertTrue(jetty92SingleRequester.toRetry(new IOException(new TimeoutException())));
|
213
|
+
|
204
214
|
// Retry SocketTimeoutException, TimeoutException and EOFException
|
205
215
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new SocketTimeoutException()));
|
206
216
|
Assert.assertTrue(jetty92SingleRequester.toRetry(new TimeoutException()));
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.9.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2018-04-
|
13
|
+
date: 2018-04-12 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
@@ -123,10 +123,10 @@ files:
|
|
123
123
|
- src/test/resources/fixtures/lists_response.json
|
124
124
|
- src/test/resources/fixtures/program_response.json
|
125
125
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
126
|
-
- classpath/embulk-input-marketo-0.6.8.jar
|
127
126
|
- classpath/embulk-base-restclient-0.5.3.jar
|
128
127
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
129
128
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
129
|
+
- classpath/embulk-input-marketo-0.6.9.alpha.jar
|
130
130
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
131
131
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
132
132
|
homepage: https://github.com/treasure-data/embulk-input-marketo
|
@@ -144,9 +144,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
144
144
|
version: '0'
|
145
145
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
146
146
|
requirements:
|
147
|
-
- - '
|
147
|
+
- - '>'
|
148
148
|
- !ruby/object:Gem::Version
|
149
|
-
version:
|
149
|
+
version: 1.3.1
|
150
150
|
requirements: []
|
151
151
|
rubyforge_project:
|
152
152
|
rubygems_version: 2.1.9
|