embulk-output-mailchimp 0.3.19 → 0.3.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c09e4eb035f05f307279f2ecf2ce11c3050b1cf6
4
- data.tar.gz: c38bcf2a895c5c87cdb8d075867fac68cda8b15c
3
+ metadata.gz: c279827f48671c120b7574d8de2c05669014e4fe
4
+ data.tar.gz: 96c677aa514f29b5588bc0d901383cda9a84b7c4
5
5
  SHA512:
6
- metadata.gz: fbe4afd20c07d7863ae67c5cd35617b12de3bcedea65b7cc271689ef65f7ca84a1d0f8c508bceac15531407739c220eb868afc78c35d773ce68608ca7291f68a
7
- data.tar.gz: 8ade98e5dde219c3e43e2711bd50c71ff23c32c38e6510235cb44c320f09c2094832e33ea4bd4a6cd88fe041c6a0f947553468889d2ae66f7797223e315fb4e0
6
+ metadata.gz: ea72fabb9264611f2146d3c64e540f7179ce76298701b29bb830417fb25dfede981353c433fc8731cc81c1b0415aa6ad8dda18bc3cab30602419f7001237b508
7
+ data.tar.gz: 1c939eb9fd3558cefa4234d07961b7c25e5aa943f1f6cf59519e8191a7abda2376d72f3453e9d71f0ff79ac7baae27af7ee87c4bac434d2ddd850e17d1c309a4
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.3.20 - 2017-10-16
2
+ - Added pagination for interest categories [#39](https://github.com/treasure-data/embulk-output-mailchimp/pull/39)
3
+ - Refactor check list id to avoid confusing when get 404 error [#38](https://github.com/treasure-data/embulk-output-mailchimp/pull/38)
4
+ - Added pagination for merge fields [#37](https://github.com/treasure-data/embulk-output-mailchimp/pull/37)
5
+ - Fixed bug duplicated email in payload JSON [#36](https://github.com/treasure-data/embulk-output-mailchimp/pull/36)
6
+
1
7
  ## 0.3.19 - 2017-10-10
2
8
  - Fixed bug can not parse invalid JSON response and added request timeout to avoid flush MailChimp API [#35](https://github.com/treasure-data/embulk-output-mailchimp/pull/35)
3
9
 
data/build.gradle CHANGED
@@ -18,7 +18,7 @@ configurations {
18
18
  provided
19
19
  }
20
20
 
21
- version = "0.3.19"
21
+ version = "0.3.20"
22
22
 
23
23
  sourceCompatibility = 1.7
24
24
  targetCompatibility = 1.7
@@ -15,6 +15,7 @@ import com.google.common.collect.Maps;
15
15
  import org.eclipse.jetty.client.HttpResponseException;
16
16
  import org.eclipse.jetty.http.HttpMethod;
17
17
  import org.embulk.config.ConfigException;
18
+ import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
18
19
  import org.embulk.output.mailchimp.helper.MailChimpHelper;
19
20
  import org.embulk.output.mailchimp.model.CategoriesResponse;
20
21
  import org.embulk.output.mailchimp.model.ErrorResponse;
@@ -56,7 +57,7 @@ public class MailChimpClient
56
57
  *
57
58
  * @param task the task
58
59
  */
59
- public MailChimpClient(final MailChimpOutputPluginDelegate.PluginTask task)
60
+ public MailChimpClient(final PluginTask task)
60
61
  {
61
62
  mailchimpEndpoint = Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION);
62
63
  this.client = new MailChimpHttpClient(task);
@@ -64,6 +65,7 @@ public class MailChimpClient
64
65
  .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
65
66
  .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
66
67
  extractDataCenter(task);
68
+ findList(task);
67
69
  }
68
70
 
69
71
  /**
@@ -75,14 +77,14 @@ public class MailChimpClient
75
77
  * @return the report response
76
78
  * @throws JsonProcessingException the json processing exception
77
79
  */
78
- ReportResponse push(final ObjectNode node, MailChimpOutputPluginDelegate.PluginTask task)
80
+ ReportResponse push(final ObjectNode node, PluginTask task)
79
81
  throws JsonProcessingException
80
82
  {
81
83
  String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
82
84
  task.getListId());
83
85
 
84
86
  JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
85
- client.avoidFlushAPI("Pushing next request");
87
+ client.avoidFlushAPI("Process next request");
86
88
 
87
89
  if (response instanceof MissingNode) {
88
90
  ReportResponse reportResponse = new ReportResponse();
@@ -123,19 +125,39 @@ public class MailChimpClient
123
125
  * @return the map
124
126
  * @throws JsonProcessingException the json processing exception
125
127
  */
126
- Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
128
+ Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
127
129
  throws JsonProcessingException
128
130
  {
129
131
  Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
130
132
  if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
131
133
  List<String> interestCategoryNames = task.getGroupingColumns().get();
132
134
 
133
- String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
134
- task.getListId());
135
+ int count = 100;
136
+ int offset = 0;
137
+ int page = 1;
138
+ boolean hasMore = true;
139
+ JsonNode response;
140
+ List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
135
141
 
136
- JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
137
- InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
138
- InterestCategoriesResponse.class);
142
+ while (hasMore) {
143
+ String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories?count={1}&offset={2}",
144
+ task.getListId(),
145
+ count,
146
+ offset);
147
+
148
+ response = client.sendRequest(endpoint, HttpMethod.GET, task);
149
+ InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
150
+ InterestCategoriesResponse.class);
151
+
152
+ allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
153
+ if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
154
+ offset = count;
155
+ page++;
156
+ }
157
+ else {
158
+ hasMore = false;
159
+ }
160
+ }
139
161
 
140
162
  Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
141
163
  {
@@ -148,7 +170,7 @@ public class MailChimpClient
148
170
 
149
171
  // Transform to a list of available category names and validate with data that user input
150
172
  ImmutableList<String> availableCategories = FluentIterable
151
- .from(interestCategoriesResponse.getCategories())
173
+ .from(allCategoriesResponse)
152
174
  .transform(function)
153
175
  .toList();
154
176
 
@@ -158,7 +180,7 @@ public class MailChimpClient
158
180
  }
159
181
  }
160
182
 
161
- for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
183
+ for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
162
184
  String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
163
185
  task.getListId(),
164
186
  categoriesResponse.getId());
@@ -183,17 +205,39 @@ public class MailChimpClient
183
205
  * @return the map
184
206
  * @throws JsonProcessingException the json processing exception
185
207
  */
186
- Map<String, MergeField> extractMergeFieldsFromList(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
208
+ Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
187
209
  {
188
- String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields",
189
- task.getListId());
190
- JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
191
- MergeFields mergeFields = mapper.treeToValue(response,
192
- MergeFields.class);
193
- return convertMergeFieldToMap(mergeFields.getMergeFields());
210
+ int count = 100;
211
+ int offset = 0;
212
+ int page = 1;
213
+ boolean hasMore = true;
214
+ List<MergeField> allMergeFields = new ArrayList<>();
215
+
216
+ while (hasMore) {
217
+ String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields?count={1}&offset={2}",
218
+ task.getListId(),
219
+ count,
220
+ offset);
221
+
222
+ JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
223
+ MergeFields mergeFields = mapper.treeToValue(response,
224
+ MergeFields.class);
225
+
226
+ allMergeFields.addAll(mergeFields.getMergeFields());
227
+
228
+ if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
229
+ offset = count;
230
+ page++;
231
+ }
232
+ else {
233
+ hasMore = false;
234
+ }
235
+ }
236
+
237
+ return convertMergeFieldToMap(allMergeFields);
194
238
  }
195
239
 
196
- private void extractDataCenter(MailChimpOutputPluginDelegate.PluginTask task)
240
+ private void extractDataCenter(final PluginTask task)
197
241
  {
198
242
  if (task.getAuthMethod() == OAUTH) {
199
243
  // Extract data center from meta data URL
@@ -225,6 +269,18 @@ public class MailChimpClient
225
269
  }
226
270
  }
227
271
 
272
+ private void findList(final PluginTask task)
273
+ {
274
+ String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
275
+ task.getListId());
276
+ try {
277
+ client.sendRequest(endpoint, HttpMethod.GET, task);
278
+ }
279
+ catch (HttpResponseException hre) {
280
+ throw new ConfigException("The `list id` could not be found.");
281
+ }
282
+ }
283
+
228
284
  private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
229
285
  {
230
286
  Function<InterestResponse, String> function = new Function<InterestResponse, String>()
@@ -257,4 +313,10 @@ public class MailChimpClient
257
313
  .toList(),
258
314
  function);
259
315
  }
316
+
317
+ private boolean hasMorePage(final int count, final int pageSize, final int page)
318
+ {
319
+ int totalPage = count / pageSize + (count % pageSize > 0 ? 1 : 0);
320
+ return page < totalPage;
321
+ }
260
322
  }
@@ -85,17 +85,13 @@ public class MailChimpHttpClient
85
85
  {
86
86
  int status = response.getStatus();
87
87
 
88
- if (status == 404) {
89
- LOG.error("Exception occurred while sending request: {}", response.getReason());
90
- throw new ConfigException("The `list id` could not be found.");
91
- }
92
-
93
88
  return status == 429 || status / 100 != 4;
94
89
  }
95
90
 
96
91
  @Override
97
92
  protected boolean isExceptionToRetry(Exception exception)
98
93
  {
94
+ LOG.error("Exception to retry.", exception);
99
95
  // This check is to make sure the exception is retryable, e.g: server not found, internal server error...
100
96
  if (exception instanceof ConfigException || exception instanceof ExecutionException) {
101
97
  return toRetry((Exception) exception.getCause());
@@ -108,7 +104,7 @@ public class MailChimpHttpClient
108
104
  return responseBody != null && !responseBody.isEmpty() ? parseJson(responseBody) : MissingNode.getInstance();
109
105
  }
110
106
  catch (Exception ex) {
111
- LOG.info("Exception occurred while sending request.");
107
+ LOG.error("Exception occurred while sending request.", ex);
112
108
  throw Throwables.propagate(ex);
113
109
  }
114
110
  }
@@ -27,9 +27,12 @@ import org.slf4j.Logger;
27
27
 
28
28
  import java.io.IOException;
29
29
  import java.util.ArrayList;
30
+ import java.util.Arrays;
30
31
  import java.util.HashMap;
32
+ import java.util.HashSet;
31
33
  import java.util.List;
32
34
  import java.util.Map;
35
+ import java.util.Set;
33
36
 
34
37
  import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
35
38
  import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
@@ -55,6 +58,8 @@ public class MailChimpRecordBuffer
55
58
  private List<JsonNode> records;
56
59
  private Map<String, Map<String, InterestResponse>> categories;
57
60
  private Map<String, MergeField> availableMergeFields;
61
+ private List<JsonNode> uniqueRecords;
62
+ private List<JsonNode> duplicatedRecords;
58
63
 
59
64
  /**
60
65
  * Instantiates a new Mail chimp abstract record buffer.
@@ -71,6 +76,8 @@ public class MailChimpRecordBuffer
71
76
  .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
72
77
  this.records = new ArrayList<>();
73
78
  this.categories = new HashMap<>();
79
+ this.uniqueRecords = new ArrayList<>();
80
+ this.duplicatedRecords = new ArrayList<>();
74
81
  this.mailChimpClient = new MailChimpClient(task);
75
82
  }
76
83
 
@@ -88,20 +95,16 @@ public class MailChimpRecordBuffer
88
95
 
89
96
  records.add(record);
90
97
  if (requestCount >= task.getMaxRecordsPerRequest()) {
91
- ObjectNode subcribers = processSubcribers(records, task);
92
- ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
98
+ filterDuplicatedRecords();
99
+ pushData();
93
100
 
94
101
  if (totalCount % 1000 == 0) {
95
102
  LOG.info("Pushed {} records", totalCount);
96
103
  }
97
104
 
98
- LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
99
- reportResponse.getTotalCreated(),
100
- reportResponse.getTotalUpdated(),
101
- reportResponse.getErrorCount());
102
- mailChimpClient.handleErrors(reportResponse.getErrors());
103
-
104
105
  records = new ArrayList<>();
106
+ uniqueRecords = new ArrayList<>();
107
+ duplicatedRecords = new ArrayList<>();
105
108
  requestCount = 0;
106
109
  }
107
110
  }
@@ -121,14 +124,11 @@ public class MailChimpRecordBuffer
121
124
  {
122
125
  try {
123
126
  if (records.size() > 0) {
124
- ObjectNode subcribers = processSubcribers(records, task);
125
- ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
126
- LOG.info("Pushed {} records", records.size());
127
- LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
128
- reportResponse.getTotalCreated(),
129
- reportResponse.getTotalUpdated(),
130
- reportResponse.getErrorCount());
131
- mailChimpClient.handleErrors(reportResponse.getErrors());
127
+ filterDuplicatedRecords();
128
+ pushData();
129
+ records = new ArrayList<>();
130
+ uniqueRecords = new ArrayList<>();
131
+ duplicatedRecords = new ArrayList<>();
132
132
  }
133
133
 
134
134
  return Exec.newTaskReport().set("pushed", totalCount);
@@ -163,8 +163,6 @@ public class MailChimpRecordBuffer
163
163
  private ObjectNode processSubcribers(final List<JsonNode> data, final PluginTask task)
164
164
  throws JsonProcessingException
165
165
  {
166
- LOG.info("Start to process subscriber data");
167
-
168
166
  // Should loop the names and get the id of interest categories.
169
167
  // The reason why we put categories validation here because we can not share data between instance.
170
168
  categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
@@ -277,4 +275,46 @@ public class MailChimpRecordBuffer
277
275
 
278
276
  return interests;
279
277
  }
278
+
279
+ private void filterDuplicatedRecords()
280
+ {
281
+ Set<String> uniqueEmails = new HashSet<>();
282
+ for (JsonNode node : records) {
283
+ if (uniqueEmails.contains(node.findPath(task.getEmailColumn()).asText())) {
284
+ duplicatedRecords.add(node);
285
+ }
286
+ else {
287
+ uniqueEmails.add(node.findPath(task.getEmailColumn()).asText());
288
+ uniqueRecords.add(node);
289
+ }
290
+ }
291
+ }
292
+
293
+ private void pushData() throws JsonProcessingException
294
+ {
295
+ long startTime = System.currentTimeMillis();
296
+ ObjectNode subscribers = processSubcribers(uniqueRecords, task);
297
+ ReportResponse reportResponse = mailChimpClient.push(subscribers, task);
298
+
299
+ LOG.info("Done with {} record(s). Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
300
+ records.size(), reportResponse.getTotalCreated(),
301
+ reportResponse.getTotalUpdated(),
302
+ reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
303
+ mailChimpClient.handleErrors(reportResponse.getErrors());
304
+
305
+ if (duplicatedRecords.size() > 0) {
306
+ LOG.info("Start to process {} duplicated record(s)", duplicatedRecords.size());
307
+ for (JsonNode duplicatedRecord : duplicatedRecords) {
308
+ startTime = System.currentTimeMillis();
309
+ subscribers = processSubcribers(Arrays.asList(duplicatedRecord), task);
310
+ reportResponse = mailChimpClient.push(subscribers, task);
311
+
312
+ LOG.info("Done. Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
313
+ reportResponse.getTotalCreated(),
314
+ reportResponse.getTotalUpdated(),
315
+ reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
316
+ mailChimpClient.handleErrors(reportResponse.getErrors());
317
+ }
318
+ }
319
+ }
280
320
  }
@@ -12,6 +12,9 @@ public class InterestCategoriesResponse
12
12
  @JsonProperty("categories")
13
13
  private List<CategoriesResponse> categories;
14
14
 
15
+ @JsonProperty("total_items")
16
+ private int totalItems;
17
+
15
18
  public List<CategoriesResponse> getCategories()
16
19
  {
17
20
  return categories;
@@ -21,4 +24,14 @@ public class InterestCategoriesResponse
21
24
  {
22
25
  this.categories = categories;
23
26
  }
27
+
28
+ public int getTotalItems()
29
+ {
30
+ return totalItems;
31
+ }
32
+
33
+ public void setTotalItems(int totalItems)
34
+ {
35
+ this.totalItems = totalItems;
36
+ }
24
37
  }
@@ -12,6 +12,9 @@ public class MergeFields
12
12
  @JsonProperty("merge_fields")
13
13
  private List<MergeField> mergeFields;
14
14
 
15
+ @JsonProperty("total_items")
16
+ private int totalItems;
17
+
15
18
  public List<MergeField> getMergeFields()
16
19
  {
17
20
  return mergeFields;
@@ -21,4 +24,14 @@ public class MergeFields
21
24
  {
22
25
  this.mergeFields = mergeFields;
23
26
  }
27
+
28
+ public int getTotalItems()
29
+ {
30
+ return totalItems;
31
+ }
32
+
33
+ public void setTotalItems(int totalItems)
34
+ {
35
+ this.totalItems = totalItems;
36
+ }
24
37
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-mailchimp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.19
4
+ version: 0.3.20
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thang Nguyen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-10-10 00:00:00.000000000 Z
11
+ date: 2017-10-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -89,7 +89,7 @@ files:
89
89
  - test/override_assert_raise.rb
90
90
  - test/run-test.rb
91
91
  - classpath/jetty-io-9.2.14.v20151106.jar
92
- - classpath/embulk-output-mailchimp-0.3.19.jar
92
+ - classpath/embulk-output-mailchimp-0.3.20.jar
93
93
  - classpath/jetty-util-9.2.14.v20151106.jar
94
94
  - classpath/jetty-http-9.2.14.v20151106.jar
95
95
  - classpath/jetty-client-9.2.14.v20151106.jar