embulk-output-mailchimp 0.3.19 → 0.3.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/build.gradle +1 -1
- data/classpath/embulk-output-mailchimp-0.3.20.jar +0 -0
- data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java +81 -19
- data/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java +2 -6
- data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java +58 -18
- data/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java +13 -0
- data/src/main/java/org/embulk/output/mailchimp/model/MergeFields.java +13 -0
- metadata +3 -3
- data/classpath/embulk-output-mailchimp-0.3.19.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c279827f48671c120b7574d8de2c05669014e4fe
|
4
|
+
data.tar.gz: 96c677aa514f29b5588bc0d901383cda9a84b7c4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ea72fabb9264611f2146d3c64e540f7179ce76298701b29bb830417fb25dfede981353c433fc8731cc81c1b0415aa6ad8dda18bc3cab30602419f7001237b508
|
7
|
+
data.tar.gz: 1c939eb9fd3558cefa4234d07961b7c25e5aa943f1f6cf59519e8191a7abda2376d72f3453e9d71f0ff79ac7baae27af7ee87c4bac434d2ddd850e17d1c309a4
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.3.20 - 2017-10-16
|
2
|
+
- Added pagination for interest categories [#39](https://github.com/treasure-data/embulk-output-mailchimp/pull/39)
|
3
|
+
- Refactor check list id to avoid confusing when get 404 error [#38](https://github.com/treasure-data/embulk-output-mailchimp/pull/38)
|
4
|
+
- Added pagination for merge fields [#37](https://github.com/treasure-data/embulk-output-mailchimp/pull/37)
|
5
|
+
- Fixed bug duplicated email in payload JSON [#36](https://github.com/treasure-data/embulk-output-mailchimp/pull/36)
|
6
|
+
|
1
7
|
## 0.3.19 - 2017-10-10
|
2
8
|
- Fixed bug can not parse invalid JSON response and added request timeout to avoid flush MailChimp API [#35](https://github.com/treasure-data/embulk-output-mailchimp/pull/35)
|
3
9
|
|
data/build.gradle
CHANGED
Binary file
|
@@ -15,6 +15,7 @@ import com.google.common.collect.Maps;
|
|
15
15
|
import org.eclipse.jetty.client.HttpResponseException;
|
16
16
|
import org.eclipse.jetty.http.HttpMethod;
|
17
17
|
import org.embulk.config.ConfigException;
|
18
|
+
import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
|
18
19
|
import org.embulk.output.mailchimp.helper.MailChimpHelper;
|
19
20
|
import org.embulk.output.mailchimp.model.CategoriesResponse;
|
20
21
|
import org.embulk.output.mailchimp.model.ErrorResponse;
|
@@ -56,7 +57,7 @@ public class MailChimpClient
|
|
56
57
|
*
|
57
58
|
* @param task the task
|
58
59
|
*/
|
59
|
-
public MailChimpClient(final
|
60
|
+
public MailChimpClient(final PluginTask task)
|
60
61
|
{
|
61
62
|
mailchimpEndpoint = Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION);
|
62
63
|
this.client = new MailChimpHttpClient(task);
|
@@ -64,6 +65,7 @@ public class MailChimpClient
|
|
64
65
|
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
|
65
66
|
.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
|
66
67
|
extractDataCenter(task);
|
68
|
+
findList(task);
|
67
69
|
}
|
68
70
|
|
69
71
|
/**
|
@@ -75,14 +77,14 @@ public class MailChimpClient
|
|
75
77
|
* @return the report response
|
76
78
|
* @throws JsonProcessingException the json processing exception
|
77
79
|
*/
|
78
|
-
ReportResponse push(final ObjectNode node,
|
80
|
+
ReportResponse push(final ObjectNode node, PluginTask task)
|
79
81
|
throws JsonProcessingException
|
80
82
|
{
|
81
83
|
String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
|
82
84
|
task.getListId());
|
83
85
|
|
84
86
|
JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
|
85
|
-
client.avoidFlushAPI("
|
87
|
+
client.avoidFlushAPI("Process next request");
|
86
88
|
|
87
89
|
if (response instanceof MissingNode) {
|
88
90
|
ReportResponse reportResponse = new ReportResponse();
|
@@ -123,19 +125,39 @@ public class MailChimpClient
|
|
123
125
|
* @return the map
|
124
126
|
* @throws JsonProcessingException the json processing exception
|
125
127
|
*/
|
126
|
-
Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final
|
128
|
+
Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
|
127
129
|
throws JsonProcessingException
|
128
130
|
{
|
129
131
|
Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
|
130
132
|
if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
|
131
133
|
List<String> interestCategoryNames = task.getGroupingColumns().get();
|
132
134
|
|
133
|
-
|
134
|
-
|
135
|
+
int count = 100;
|
136
|
+
int offset = 0;
|
137
|
+
int page = 1;
|
138
|
+
boolean hasMore = true;
|
139
|
+
JsonNode response;
|
140
|
+
List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
|
135
141
|
|
136
|
-
|
137
|
-
|
138
|
-
|
142
|
+
while (hasMore) {
|
143
|
+
String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories?count={1}&offset={2}",
|
144
|
+
task.getListId(),
|
145
|
+
count,
|
146
|
+
offset);
|
147
|
+
|
148
|
+
response = client.sendRequest(endpoint, HttpMethod.GET, task);
|
149
|
+
InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
|
150
|
+
InterestCategoriesResponse.class);
|
151
|
+
|
152
|
+
allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
|
153
|
+
if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
|
154
|
+
offset = count;
|
155
|
+
page++;
|
156
|
+
}
|
157
|
+
else {
|
158
|
+
hasMore = false;
|
159
|
+
}
|
160
|
+
}
|
139
161
|
|
140
162
|
Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
|
141
163
|
{
|
@@ -148,7 +170,7 @@ public class MailChimpClient
|
|
148
170
|
|
149
171
|
// Transform to a list of available category names and validate with data that user input
|
150
172
|
ImmutableList<String> availableCategories = FluentIterable
|
151
|
-
.from(
|
173
|
+
.from(allCategoriesResponse)
|
152
174
|
.transform(function)
|
153
175
|
.toList();
|
154
176
|
|
@@ -158,7 +180,7 @@ public class MailChimpClient
|
|
158
180
|
}
|
159
181
|
}
|
160
182
|
|
161
|
-
for (CategoriesResponse categoriesResponse :
|
183
|
+
for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
|
162
184
|
String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
|
163
185
|
task.getListId(),
|
164
186
|
categoriesResponse.getId());
|
@@ -183,17 +205,39 @@ public class MailChimpClient
|
|
183
205
|
* @return the map
|
184
206
|
* @throws JsonProcessingException the json processing exception
|
185
207
|
*/
|
186
|
-
Map<String, MergeField> extractMergeFieldsFromList(
|
208
|
+
Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
|
187
209
|
{
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
210
|
+
int count = 100;
|
211
|
+
int offset = 0;
|
212
|
+
int page = 1;
|
213
|
+
boolean hasMore = true;
|
214
|
+
List<MergeField> allMergeFields = new ArrayList<>();
|
215
|
+
|
216
|
+
while (hasMore) {
|
217
|
+
String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields?count={1}&offset={2}",
|
218
|
+
task.getListId(),
|
219
|
+
count,
|
220
|
+
offset);
|
221
|
+
|
222
|
+
JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
|
223
|
+
MergeFields mergeFields = mapper.treeToValue(response,
|
224
|
+
MergeFields.class);
|
225
|
+
|
226
|
+
allMergeFields.addAll(mergeFields.getMergeFields());
|
227
|
+
|
228
|
+
if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
|
229
|
+
offset = count;
|
230
|
+
page++;
|
231
|
+
}
|
232
|
+
else {
|
233
|
+
hasMore = false;
|
234
|
+
}
|
235
|
+
}
|
236
|
+
|
237
|
+
return convertMergeFieldToMap(allMergeFields);
|
194
238
|
}
|
195
239
|
|
196
|
-
private void extractDataCenter(
|
240
|
+
private void extractDataCenter(final PluginTask task)
|
197
241
|
{
|
198
242
|
if (task.getAuthMethod() == OAUTH) {
|
199
243
|
// Extract data center from meta data URL
|
@@ -225,6 +269,18 @@ public class MailChimpClient
|
|
225
269
|
}
|
226
270
|
}
|
227
271
|
|
272
|
+
private void findList(final PluginTask task)
|
273
|
+
{
|
274
|
+
String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
|
275
|
+
task.getListId());
|
276
|
+
try {
|
277
|
+
client.sendRequest(endpoint, HttpMethod.GET, task);
|
278
|
+
}
|
279
|
+
catch (HttpResponseException hre) {
|
280
|
+
throw new ConfigException("The `list id` could not be found.");
|
281
|
+
}
|
282
|
+
}
|
283
|
+
|
228
284
|
private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
|
229
285
|
{
|
230
286
|
Function<InterestResponse, String> function = new Function<InterestResponse, String>()
|
@@ -257,4 +313,10 @@ public class MailChimpClient
|
|
257
313
|
.toList(),
|
258
314
|
function);
|
259
315
|
}
|
316
|
+
|
317
|
+
private boolean hasMorePage(final int count, final int pageSize, final int page)
|
318
|
+
{
|
319
|
+
int totalPage = count / pageSize + (count % pageSize > 0 ? 1 : 0);
|
320
|
+
return page < totalPage;
|
321
|
+
}
|
260
322
|
}
|
@@ -85,17 +85,13 @@ public class MailChimpHttpClient
|
|
85
85
|
{
|
86
86
|
int status = response.getStatus();
|
87
87
|
|
88
|
-
if (status == 404) {
|
89
|
-
LOG.error("Exception occurred while sending request: {}", response.getReason());
|
90
|
-
throw new ConfigException("The `list id` could not be found.");
|
91
|
-
}
|
92
|
-
|
93
88
|
return status == 429 || status / 100 != 4;
|
94
89
|
}
|
95
90
|
|
96
91
|
@Override
|
97
92
|
protected boolean isExceptionToRetry(Exception exception)
|
98
93
|
{
|
94
|
+
LOG.error("Exception to retry.", exception);
|
99
95
|
// This check is to make sure the exception is retryable, e.g: server not found, internal server error...
|
100
96
|
if (exception instanceof ConfigException || exception instanceof ExecutionException) {
|
101
97
|
return toRetry((Exception) exception.getCause());
|
@@ -108,7 +104,7 @@ public class MailChimpHttpClient
|
|
108
104
|
return responseBody != null && !responseBody.isEmpty() ? parseJson(responseBody) : MissingNode.getInstance();
|
109
105
|
}
|
110
106
|
catch (Exception ex) {
|
111
|
-
LOG.
|
107
|
+
LOG.error("Exception occurred while sending request.", ex);
|
112
108
|
throw Throwables.propagate(ex);
|
113
109
|
}
|
114
110
|
}
|
@@ -27,9 +27,12 @@ import org.slf4j.Logger;
|
|
27
27
|
|
28
28
|
import java.io.IOException;
|
29
29
|
import java.util.ArrayList;
|
30
|
+
import java.util.Arrays;
|
30
31
|
import java.util.HashMap;
|
32
|
+
import java.util.HashSet;
|
31
33
|
import java.util.List;
|
32
34
|
import java.util.Map;
|
35
|
+
import java.util.Set;
|
33
36
|
|
34
37
|
import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
|
35
38
|
import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
|
@@ -55,6 +58,8 @@ public class MailChimpRecordBuffer
|
|
55
58
|
private List<JsonNode> records;
|
56
59
|
private Map<String, Map<String, InterestResponse>> categories;
|
57
60
|
private Map<String, MergeField> availableMergeFields;
|
61
|
+
private List<JsonNode> uniqueRecords;
|
62
|
+
private List<JsonNode> duplicatedRecords;
|
58
63
|
|
59
64
|
/**
|
60
65
|
* Instantiates a new Mail chimp abstract record buffer.
|
@@ -71,6 +76,8 @@ public class MailChimpRecordBuffer
|
|
71
76
|
.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
|
72
77
|
this.records = new ArrayList<>();
|
73
78
|
this.categories = new HashMap<>();
|
79
|
+
this.uniqueRecords = new ArrayList<>();
|
80
|
+
this.duplicatedRecords = new ArrayList<>();
|
74
81
|
this.mailChimpClient = new MailChimpClient(task);
|
75
82
|
}
|
76
83
|
|
@@ -88,20 +95,16 @@ public class MailChimpRecordBuffer
|
|
88
95
|
|
89
96
|
records.add(record);
|
90
97
|
if (requestCount >= task.getMaxRecordsPerRequest()) {
|
91
|
-
|
92
|
-
|
98
|
+
filterDuplicatedRecords();
|
99
|
+
pushData();
|
93
100
|
|
94
101
|
if (totalCount % 1000 == 0) {
|
95
102
|
LOG.info("Pushed {} records", totalCount);
|
96
103
|
}
|
97
104
|
|
98
|
-
LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
|
99
|
-
reportResponse.getTotalCreated(),
|
100
|
-
reportResponse.getTotalUpdated(),
|
101
|
-
reportResponse.getErrorCount());
|
102
|
-
mailChimpClient.handleErrors(reportResponse.getErrors());
|
103
|
-
|
104
105
|
records = new ArrayList<>();
|
106
|
+
uniqueRecords = new ArrayList<>();
|
107
|
+
duplicatedRecords = new ArrayList<>();
|
105
108
|
requestCount = 0;
|
106
109
|
}
|
107
110
|
}
|
@@ -121,14 +124,11 @@ public class MailChimpRecordBuffer
|
|
121
124
|
{
|
122
125
|
try {
|
123
126
|
if (records.size() > 0) {
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
reportResponse.getTotalUpdated(),
|
130
|
-
reportResponse.getErrorCount());
|
131
|
-
mailChimpClient.handleErrors(reportResponse.getErrors());
|
127
|
+
filterDuplicatedRecords();
|
128
|
+
pushData();
|
129
|
+
records = new ArrayList<>();
|
130
|
+
uniqueRecords = new ArrayList<>();
|
131
|
+
duplicatedRecords = new ArrayList<>();
|
132
132
|
}
|
133
133
|
|
134
134
|
return Exec.newTaskReport().set("pushed", totalCount);
|
@@ -163,8 +163,6 @@ public class MailChimpRecordBuffer
|
|
163
163
|
private ObjectNode processSubcribers(final List<JsonNode> data, final PluginTask task)
|
164
164
|
throws JsonProcessingException
|
165
165
|
{
|
166
|
-
LOG.info("Start to process subscriber data");
|
167
|
-
|
168
166
|
// Should loop the names and get the id of interest categories.
|
169
167
|
// The reason why we put categories validation here because we can not share data between instance.
|
170
168
|
categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
|
@@ -277,4 +275,46 @@ public class MailChimpRecordBuffer
|
|
277
275
|
|
278
276
|
return interests;
|
279
277
|
}
|
278
|
+
|
279
|
+
private void filterDuplicatedRecords()
|
280
|
+
{
|
281
|
+
Set<String> uniqueEmails = new HashSet<>();
|
282
|
+
for (JsonNode node : records) {
|
283
|
+
if (uniqueEmails.contains(node.findPath(task.getEmailColumn()).asText())) {
|
284
|
+
duplicatedRecords.add(node);
|
285
|
+
}
|
286
|
+
else {
|
287
|
+
uniqueEmails.add(node.findPath(task.getEmailColumn()).asText());
|
288
|
+
uniqueRecords.add(node);
|
289
|
+
}
|
290
|
+
}
|
291
|
+
}
|
292
|
+
|
293
|
+
private void pushData() throws JsonProcessingException
|
294
|
+
{
|
295
|
+
long startTime = System.currentTimeMillis();
|
296
|
+
ObjectNode subscribers = processSubcribers(uniqueRecords, task);
|
297
|
+
ReportResponse reportResponse = mailChimpClient.push(subscribers, task);
|
298
|
+
|
299
|
+
LOG.info("Done with {} record(s). Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
|
300
|
+
records.size(), reportResponse.getTotalCreated(),
|
301
|
+
reportResponse.getTotalUpdated(),
|
302
|
+
reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
|
303
|
+
mailChimpClient.handleErrors(reportResponse.getErrors());
|
304
|
+
|
305
|
+
if (duplicatedRecords.size() > 0) {
|
306
|
+
LOG.info("Start to process {} duplicated record(s)", duplicatedRecords.size());
|
307
|
+
for (JsonNode duplicatedRecord : duplicatedRecords) {
|
308
|
+
startTime = System.currentTimeMillis();
|
309
|
+
subscribers = processSubcribers(Arrays.asList(duplicatedRecord), task);
|
310
|
+
reportResponse = mailChimpClient.push(subscribers, task);
|
311
|
+
|
312
|
+
LOG.info("Done. Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
|
313
|
+
reportResponse.getTotalCreated(),
|
314
|
+
reportResponse.getTotalUpdated(),
|
315
|
+
reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
|
316
|
+
mailChimpClient.handleErrors(reportResponse.getErrors());
|
317
|
+
}
|
318
|
+
}
|
319
|
+
}
|
280
320
|
}
|
@@ -12,6 +12,9 @@ public class InterestCategoriesResponse
|
|
12
12
|
@JsonProperty("categories")
|
13
13
|
private List<CategoriesResponse> categories;
|
14
14
|
|
15
|
+
@JsonProperty("total_items")
|
16
|
+
private int totalItems;
|
17
|
+
|
15
18
|
public List<CategoriesResponse> getCategories()
|
16
19
|
{
|
17
20
|
return categories;
|
@@ -21,4 +24,14 @@ public class InterestCategoriesResponse
|
|
21
24
|
{
|
22
25
|
this.categories = categories;
|
23
26
|
}
|
27
|
+
|
28
|
+
public int getTotalItems()
|
29
|
+
{
|
30
|
+
return totalItems;
|
31
|
+
}
|
32
|
+
|
33
|
+
public void setTotalItems(int totalItems)
|
34
|
+
{
|
35
|
+
this.totalItems = totalItems;
|
36
|
+
}
|
24
37
|
}
|
@@ -12,6 +12,9 @@ public class MergeFields
|
|
12
12
|
@JsonProperty("merge_fields")
|
13
13
|
private List<MergeField> mergeFields;
|
14
14
|
|
15
|
+
@JsonProperty("total_items")
|
16
|
+
private int totalItems;
|
17
|
+
|
15
18
|
public List<MergeField> getMergeFields()
|
16
19
|
{
|
17
20
|
return mergeFields;
|
@@ -21,4 +24,14 @@ public class MergeFields
|
|
21
24
|
{
|
22
25
|
this.mergeFields = mergeFields;
|
23
26
|
}
|
27
|
+
|
28
|
+
public int getTotalItems()
|
29
|
+
{
|
30
|
+
return totalItems;
|
31
|
+
}
|
32
|
+
|
33
|
+
public void setTotalItems(int totalItems)
|
34
|
+
{
|
35
|
+
this.totalItems = totalItems;
|
36
|
+
}
|
24
37
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-mailchimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thang Nguyen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-10-
|
11
|
+
date: 2017-10-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -89,7 +89,7 @@ files:
|
|
89
89
|
- test/override_assert_raise.rb
|
90
90
|
- test/run-test.rb
|
91
91
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
92
|
-
- classpath/embulk-output-mailchimp-0.3.
|
92
|
+
- classpath/embulk-output-mailchimp-0.3.20.jar
|
93
93
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
94
94
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
95
95
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
Binary file
|