embulk-output-mailchimp 0.3.25 → 0.3.28
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java +49 -13
- data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java +17 -0
- data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java +37 -30
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b429373deb2a79bd444d223f70818c7a7517bd63
|
4
|
+
data.tar.gz: 06fb08e83602804217e386754e39cd7cb542c2e8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b096372c93fcb6a0c2930d0b98ade33a1d52277f65fd1554e6d80178ee2179b300847c69c907d4e2cbb1279f7028e6eac3804d123f8e92681e1955182df5bc22
|
7
|
+
data.tar.gz: efd31474670848d2457b2fdc5ea68b990902971543e884dd473fcb57064b34687a79beb95188bc06349e5448cac1a4919b91cadc90083a0fe39cca32d09c5c60
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
## 0.3.28 - 2018-10-29
|
2
|
+
|
3
|
+
- Quick fix a typo
|
4
|
+
|
5
|
+
## 0.3.26 - 2018-10-29
|
6
|
+
|
7
|
+
- Fix an NPE when there isn't a column corresponds to a configured group ('category')
|
8
|
+
|
1
9
|
## 0.3.25 - 2018-10-11
|
2
10
|
|
3
11
|
- Fixed an NPE when column names and merge tags are not exactly (case-sensitive) matched
|
data/build.gradle
CHANGED
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
|
9
9
|
import com.fasterxml.jackson.databind.node.NullNode;
|
10
10
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
11
11
|
import com.google.common.base.Function;
|
12
|
+
import com.google.common.base.Optional;
|
12
13
|
import com.google.common.base.Throwables;
|
13
14
|
import com.google.common.collect.FluentIterable;
|
14
15
|
import org.embulk.base.restclient.jackson.JacksonServiceRecord;
|
@@ -25,6 +26,8 @@ import org.embulk.spi.Exec;
|
|
25
26
|
import org.embulk.spi.Schema;
|
26
27
|
import org.slf4j.Logger;
|
27
28
|
|
29
|
+
import javax.annotation.Nullable;
|
30
|
+
|
28
31
|
import java.io.IOException;
|
29
32
|
import java.util.ArrayList;
|
30
33
|
import java.util.Arrays;
|
@@ -34,11 +37,14 @@ import java.util.List;
|
|
34
37
|
import java.util.Map;
|
35
38
|
import java.util.Set;
|
36
39
|
import java.util.TreeMap;
|
40
|
+
import java.util.TreeSet;
|
37
41
|
|
42
|
+
import static com.google.common.base.Joiner.on;
|
38
43
|
import static java.lang.String.CASE_INSENSITIVE_ORDER;
|
39
44
|
import static java.lang.String.format;
|
40
45
|
import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
|
41
46
|
import static org.embulk.output.mailchimp.helper.MailChimpHelper.fromCommaSeparatedString;
|
47
|
+
import static org.embulk.output.mailchimp.helper.MailChimpHelper.jsonGetIgnoreCase;
|
42
48
|
import static org.embulk.output.mailchimp.helper.MailChimpHelper.orderJsonNode;
|
43
49
|
import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
|
44
50
|
import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
|
@@ -168,6 +174,13 @@ public class MailChimpRecordBuffer
|
|
168
174
|
// The reason why we put categories validation here because we can not share data between instance.
|
169
175
|
if (categories == null) {
|
170
176
|
categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
|
177
|
+
|
178
|
+
Set<String> categoriesNames = categories.keySet();
|
179
|
+
Set<String> columnNames = caseInsensitiveColumnNames();
|
180
|
+
if (!columnNames.containsAll(categoriesNames)) {
|
181
|
+
categoriesNames.removeAll(columnNames);
|
182
|
+
LOG.warn("Data column for category '{}' could not be found", on(", ").join(categoriesNames));
|
183
|
+
}
|
171
184
|
}
|
172
185
|
|
173
186
|
// Extract merge fields detail
|
@@ -249,7 +262,7 @@ public class MailChimpRecordBuffer
|
|
249
262
|
|
250
263
|
// Update interest categories if exist
|
251
264
|
if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
|
252
|
-
property.set("interests", buildInterestCategories(
|
265
|
+
property.set("interests", buildInterestCategories(input));
|
253
266
|
}
|
254
267
|
|
255
268
|
// Update language if exist
|
@@ -262,31 +275,36 @@ public class MailChimpRecordBuffer
|
|
262
275
|
};
|
263
276
|
}
|
264
277
|
|
265
|
-
private ObjectNode buildInterestCategories(final
|
278
|
+
private ObjectNode buildInterestCategories(final JsonNode input)
|
266
279
|
{
|
267
280
|
ObjectNode interests = JsonNodeFactory.instance.objectNode();
|
268
281
|
|
269
282
|
if (task.getGroupingColumns().isPresent()) {
|
270
283
|
for (String category : task.getGroupingColumns().get()) {
|
271
|
-
|
272
|
-
|
273
|
-
|
284
|
+
Optional<JsonNode> inputValue = jsonGetIgnoreCase(input, category);
|
285
|
+
if (!inputValue.isPresent()) {
|
286
|
+
// Silently ignore if the grouping column is absent
|
287
|
+
continue;
|
288
|
+
}
|
289
|
+
List<String> recordInterests = fromCommaSeparatedString(inputValue.get().asText());
|
290
|
+
// `categories` is guaranteed to contain the `category` as it already did an early check
|
291
|
+
Map<String, InterestResponse> availableInterests = categories.get(category);
|
274
292
|
|
275
293
|
// Only update user-predefined categories if replace interests != true
|
276
294
|
if (!task.getReplaceInterests()) {
|
277
|
-
for (String
|
278
|
-
if (
|
279
|
-
interests.put(
|
295
|
+
for (String recordInterest : recordInterests) {
|
296
|
+
if (availableInterests.get(recordInterest) != null) {
|
297
|
+
interests.put(availableInterests.get(recordInterest).getId(), true);
|
280
298
|
}
|
281
299
|
}
|
282
300
|
} // Otherwise, force update all categories include user-predefined categories
|
283
301
|
else if (task.getReplaceInterests()) {
|
284
|
-
for (String
|
285
|
-
if (
|
286
|
-
interests.put(
|
302
|
+
for (String availableInterest : availableInterests.keySet()) {
|
303
|
+
if (recordInterests.contains(availableInterest)) {
|
304
|
+
interests.put(availableInterests.get(availableInterest).getId(), true);
|
287
305
|
}
|
288
306
|
else {
|
289
|
-
interests.put(
|
307
|
+
interests.put(availableInterests.get(availableInterest).getId(), false);
|
290
308
|
}
|
291
309
|
}
|
292
310
|
}
|
@@ -313,7 +331,8 @@ public class MailChimpRecordBuffer
|
|
313
331
|
private void pushData() throws JsonProcessingException
|
314
332
|
{
|
315
333
|
long startTime = System.currentTimeMillis();
|
316
|
-
ObjectNode subscribers =
|
334
|
+
ObjectNode subscribers =
|
335
|
+
processSubcribers(uniqueRecords, task);
|
317
336
|
ReportResponse reportResponse = mailChimpClient.push(subscribers, task);
|
318
337
|
|
319
338
|
LOG.info("Done with {} record(s). Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
|
@@ -340,4 +359,21 @@ public class MailChimpRecordBuffer
|
|
340
359
|
}
|
341
360
|
}
|
342
361
|
}
|
362
|
+
|
363
|
+
private Set<String> caseInsensitiveColumnNames()
|
364
|
+
{
|
365
|
+
Set<String> columns = new TreeSet<>(CASE_INSENSITIVE_ORDER);
|
366
|
+
columns.addAll(FluentIterable
|
367
|
+
.from(schema.getColumns())
|
368
|
+
.transform(new Function<Column, String>() {
|
369
|
+
@Nullable
|
370
|
+
@Override
|
371
|
+
public String apply(@Nullable Column col)
|
372
|
+
{
|
373
|
+
return col.getName();
|
374
|
+
}
|
375
|
+
})
|
376
|
+
.toSet());
|
377
|
+
return columns;
|
378
|
+
}
|
343
379
|
}
|
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|
7
7
|
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
8
8
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
9
9
|
import com.google.common.base.Function;
|
10
|
+
import com.google.common.base.Optional;
|
10
11
|
import com.google.common.base.Splitter;
|
11
12
|
import com.google.common.collect.Lists;
|
12
13
|
import com.google.common.collect.Multimap;
|
@@ -16,8 +17,11 @@ import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
|
|
16
17
|
import javax.annotation.Nullable;
|
17
18
|
|
18
19
|
import java.io.IOException;
|
20
|
+
import java.util.Iterator;
|
19
21
|
import java.util.List;
|
20
22
|
|
23
|
+
import static java.util.Objects.requireNonNull;
|
24
|
+
|
21
25
|
/**
|
22
26
|
* Created by thangnc on 4/26/17.
|
23
27
|
*/
|
@@ -125,4 +129,17 @@ public final class MailChimpHelper
|
|
125
129
|
|
126
130
|
return orderedNode;
|
127
131
|
}
|
132
|
+
|
133
|
+
public static Optional<JsonNode> jsonGetIgnoreCase(JsonNode node, String fieldName)
|
134
|
+
{
|
135
|
+
requireNonNull(fieldName);
|
136
|
+
Iterator<String> fieldNames = node.fieldNames();
|
137
|
+
while (fieldNames.hasNext()) {
|
138
|
+
String curFieldName = fieldNames.next();
|
139
|
+
if (fieldName.equalsIgnoreCase((curFieldName))) {
|
140
|
+
return Optional.of(node.get(curFieldName));
|
141
|
+
}
|
142
|
+
}
|
143
|
+
return Optional.absent();
|
144
|
+
}
|
128
145
|
}
|
@@ -24,6 +24,7 @@ import java.text.MessageFormat;
|
|
24
24
|
import java.util.concurrent.ExecutionException;
|
25
25
|
import java.util.concurrent.TimeoutException;
|
26
26
|
|
27
|
+
import static com.google.common.base.Throwables.propagate;
|
27
28
|
import static org.eclipse.jetty.http.HttpHeader.AUTHORIZATION;
|
28
29
|
import static org.eclipse.jetty.http.HttpMethod.GET;
|
29
30
|
import static org.eclipse.jetty.http.HttpMethod.POST;
|
@@ -64,41 +65,47 @@ public class MailChimpRetryable implements AutoCloseable
|
|
64
65
|
|
65
66
|
private String sendRequest(final String path, final StringContentProvider contentProvider)
|
66
67
|
{
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
@Override
|
72
|
-
public void requestOnce(HttpClient client, Response.Listener responseListener)
|
68
|
+
try {
|
69
|
+
return retryHelper.requestWithRetry(
|
70
|
+
new StringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
|
71
|
+
new Jetty92SingleRequester()
|
73
72
|
{
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
73
|
+
@Override
|
74
|
+
public void requestOnce(HttpClient client, Response.Listener responseListener)
|
75
|
+
{
|
76
|
+
createTokenHolder(client);
|
77
|
+
Request request = client.newRequest(tokenHolder.getEndpoint() + path)
|
78
|
+
.header(AUTHORIZATION, authorizationHeader)
|
79
|
+
.method(GET);
|
80
|
+
if (contentProvider != null) {
|
81
|
+
request = request.method(POST).content(contentProvider);
|
82
|
+
}
|
83
|
+
request.send(responseListener);
|
80
84
|
}
|
81
|
-
request.send(responseListener);
|
82
|
-
}
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
86
|
+
@Override
|
87
|
+
protected boolean isResponseStatusToRetry(Response response)
|
88
|
+
{
|
89
|
+
// Retry if it's a server or rate limit exceeded error
|
90
|
+
return (response.getStatus() != 500 && response.getStatus() / 100 != 4) || response.getStatus() == 429;
|
91
|
+
}
|
90
92
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
93
|
+
@Override
|
94
|
+
protected boolean isExceptionToRetry(Exception exception)
|
95
|
+
{
|
96
|
+
// This check is to make sure if the original exception is retryable, i.e.
|
97
|
+
// server not found, internal server error...
|
98
|
+
if (exception instanceof ConfigException || exception instanceof ExecutionException) {
|
99
|
+
return toRetry((Exception) exception.getCause());
|
100
|
+
}
|
101
|
+
return exception instanceof TimeoutException || super.isExceptionToRetry(exception);
|
98
102
|
}
|
99
|
-
|
100
|
-
|
101
|
-
|
103
|
+
});
|
104
|
+
}
|
105
|
+
catch (HttpResponseException ex) {
|
106
|
+
LOG.error("Unexpected response from request to {}", path, ex);
|
107
|
+
throw propagate(ex);
|
108
|
+
}
|
102
109
|
}
|
103
110
|
|
104
111
|
@Override
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-mailchimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.28
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thang Nguyen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,12 +89,12 @@ files:
|
|
89
89
|
- test/override_assert_raise.rb
|
90
90
|
- test/run-test.rb
|
91
91
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
92
|
-
- classpath/embulk-output-mailchimp-0.3.25.jar
|
93
92
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
94
93
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
95
94
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
96
95
|
- classpath/embulk-base-restclient-0.5.3.jar
|
97
96
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
97
|
+
- classpath/embulk-output-mailchimp-0.3.28.jar
|
98
98
|
homepage: https://github.com/treasure-data/embulk-output-mailchimp
|
99
99
|
licenses:
|
100
100
|
- MIT
|