embulk-output-mailchimp 0.3.19 → 0.3.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c09e4eb035f05f307279f2ecf2ce11c3050b1cf6
-  data.tar.gz: c38bcf2a895c5c87cdb8d075867fac68cda8b15c
+  metadata.gz: c279827f48671c120b7574d8de2c05669014e4fe
+  data.tar.gz: 96c677aa514f29b5588bc0d901383cda9a84b7c4
 SHA512:
-  metadata.gz: fbe4afd20c07d7863ae67c5cd35617b12de3bcedea65b7cc271689ef65f7ca84a1d0f8c508bceac15531407739c220eb868afc78c35d773ce68608ca7291f68a
-  data.tar.gz: 8ade98e5dde219c3e43e2711bd50c71ff23c32c38e6510235cb44c320f09c2094832e33ea4bd4a6cd88fe041c6a0f947553468889d2ae66f7797223e315fb4e0
+  metadata.gz: ea72fabb9264611f2146d3c64e540f7179ce76298701b29bb830417fb25dfede981353c433fc8731cc81c1b0415aa6ad8dda18bc3cab30602419f7001237b508
+  data.tar.gz: 1c939eb9fd3558cefa4234d07961b7c25e5aa943f1f6cf59519e8191a7abda2376d72f3453e9d71f0ff79ac7baae27af7ee87c4bac434d2ddd850e17d1c309a4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.20 - 2017-10-16
+- Added pagination for interest categories [#39](https://github.com/treasure-data/embulk-output-mailchimp/pull/39)
+- Refactor check list id to avoid confusing when get 404 error [#38](https://github.com/treasure-data/embulk-output-mailchimp/pull/38)
+- Added pagination for merge fields [#37](https://github.com/treasure-data/embulk-output-mailchimp/pull/37)
+- Fixed bug duplicated email in payload JSON [#36](https://github.com/treasure-data/embulk-output-mailchimp/pull/36)
+
 ## 0.3.19 - 2017-10-10
 - Fixed bug can not parse invalid JSON response and added request timeout to avoid flush MailChimp API [#35](https://github.com/treasure-data/embulk-output-mailchimp/pull/35)
 

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.19"
+version = "0.3.20"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java

@@ -15,6 +15,7 @@ import com.google.common.collect.Maps;
 import org.eclipse.jetty.client.HttpResponseException;
 import org.eclipse.jetty.http.HttpMethod;
 import org.embulk.config.ConfigException;
+import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
 import org.embulk.output.mailchimp.helper.MailChimpHelper;
 import org.embulk.output.mailchimp.model.CategoriesResponse;
 import org.embulk.output.mailchimp.model.ErrorResponse;
@@ -56,7 +57,7 @@ public class MailChimpClient
      *
      * @param task the task
      */
-    public MailChimpClient(final MailChimpOutputPluginDelegate.PluginTask task)
+    public MailChimpClient(final PluginTask task)
     {
         mailchimpEndpoint = Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION);
         this.client = new MailChimpHttpClient(task);
@@ -64,6 +65,7 @@ public class MailChimpClient
                 .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
                 .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
         extractDataCenter(task);
+        findList(task);
     }
 
     /**
@@ -75,14 +77,14 @@ public class MailChimpClient
      * @return the report response
      * @throws JsonProcessingException the json processing exception
      */
-    ReportResponse push(final ObjectNode node, MailChimpOutputPluginDelegate.PluginTask task)
+    ReportResponse push(final ObjectNode node, PluginTask task)
             throws JsonProcessingException
     {
         String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
                                                task.getListId());
 
         JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
-        client.avoidFlushAPI("Pushing next request");
+        client.avoidFlushAPI("Process next request");
 
         if (response instanceof MissingNode) {
             ReportResponse reportResponse = new ReportResponse();
@@ -123,19 +125,39 @@ public class MailChimpClient
      * @return the map
      * @throws JsonProcessingException the json processing exception
      */
-    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
+    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
             throws JsonProcessingException
     {
         Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
         if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
             List<String> interestCategoryNames = task.getGroupingColumns().get();
 
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
-                                                   task.getListId());
+            int count = 100;
+            int offset = 0;
+            int page = 1;
+            boolean hasMore = true;
+            JsonNode response;
+            List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
 
-            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-            InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
-                                                                                       InterestCategoriesResponse.class);
+            while (hasMore) {
+                String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories?count={1}&offset={2}",
+                                                       task.getListId(),
+                                                       count,
+                                                       offset);
+
+                response = client.sendRequest(endpoint, HttpMethod.GET, task);
+                InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
+                                                                                           InterestCategoriesResponse.class);
+
+                allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
+                if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
+                    offset = count;
+                    page++;
+                }
+                else {
+                    hasMore = false;
+                }
+            }
 
             Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
             {
@@ -148,7 +170,7 @@ public class MailChimpClient
 
             // Transform to a list of available category names and validate with data that user input
             ImmutableList<String> availableCategories = FluentIterable
-                    .from(interestCategoriesResponse.getCategories())
+                    .from(allCategoriesResponse)
                     .transform(function)
                     .toList();
 
@@ -158,7 +180,7 @@ public class MailChimpClient
                 }
             }
 
-            for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
+            for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
                 String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
                                                              task.getListId(),
                                                              categoriesResponse.getId());
@@ -183,17 +205,39 @@ public class MailChimpClient
      * @return the map
      * @throws JsonProcessingException the json processing exception
      */
-    Map<String, MergeField> extractMergeFieldsFromList(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
     {
-        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields",
-                                               task.getListId());
-        JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-        MergeFields mergeFields = mapper.treeToValue(response,
-                                                     MergeFields.class);
-        return convertMergeFieldToMap(mergeFields.getMergeFields());
+        int count = 100;
+        int offset = 0;
+        int page = 1;
+        boolean hasMore = true;
+        List<MergeField> allMergeFields = new ArrayList<>();
+
+        while (hasMore) {
+            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields?count={1}&offset={2}",
+                                                   task.getListId(),
+                                                   count,
+                                                   offset);
+
+            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
+            MergeFields mergeFields = mapper.treeToValue(response,
+                                                         MergeFields.class);
+
+            allMergeFields.addAll(mergeFields.getMergeFields());
+
+            if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
+                offset = count;
+                page++;
+            }
+            else {
+                hasMore = false;
+            }
+        }
+
+        return convertMergeFieldToMap(allMergeFields);
     }
 
-    private void extractDataCenter(MailChimpOutputPluginDelegate.PluginTask task)
+    private void extractDataCenter(final PluginTask task)
     {
         if (task.getAuthMethod() == OAUTH) {
             // Extract data center from meta data URL
@@ -225,6 +269,18 @@ public class MailChimpClient
         }
     }
 
+    private void findList(final PluginTask task)
+    {
+        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
+                                               task.getListId());
+        try {
+            client.sendRequest(endpoint, HttpMethod.GET, task);
+        }
+        catch (HttpResponseException hre) {
+            throw new ConfigException("The `list id` could not be found.");
+        }
+    }
+
     private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
     {
         Function<InterestResponse, String> function = new Function<InterestResponse, String>()
@@ -257,4 +313,10 @@ public class MailChimpClient
                                         .toList(),
                                 function);
     }
+
+    private boolean hasMorePage(final int count, final int pageSize, final int page)
+    {
+        int totalPage = count / pageSize + (count % pageSize > 0 ? 1 : 0);
+        return page < totalPage;
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java

@@ -85,17 +85,13 @@ public class MailChimpHttpClient
                         {
                             int status = response.getStatus();
 
-                            if (status == 404) {
-                                LOG.error("Exception occurred while sending request: {}", response.getReason());
-                                throw new ConfigException("The `list id` could not be found.");
-                            }
-
                             return status == 429 || status / 100 != 4;
                         }
 
                         @Override
                         protected boolean isExceptionToRetry(Exception exception)
                         {
+                            LOG.error("Exception to retry.", exception);
                             // This check is to make sure the exception is retryable, e.g: server not found, internal server error...
                             if (exception instanceof ConfigException || exception instanceof ExecutionException) {
                                 return toRetry((Exception) exception.getCause());
@@ -108,7 +104,7 @@ public class MailChimpHttpClient
             return responseBody != null && !responseBody.isEmpty() ? parseJson(responseBody) : MissingNode.getInstance();
         }
         catch (Exception ex) {
-            LOG.info("Exception occurred while sending request.");
+            LOG.error("Exception occurred while sending request.", ex);
             throw Throwables.propagate(ex);
         }
     }

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -27,9 +27,12 @@ import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
@@ -55,6 +58,8 @@ public class MailChimpRecordBuffer
     private List<JsonNode> records;
     private Map<String, Map<String, InterestResponse>> categories;
     private Map<String, MergeField> availableMergeFields;
+    private List<JsonNode> uniqueRecords;
+    private List<JsonNode> duplicatedRecords;
 
     /**
      * Instantiates a new Mail chimp abstract record buffer.
@@ -71,6 +76,8 @@ public class MailChimpRecordBuffer
                 .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
         this.records = new ArrayList<>();
         this.categories = new HashMap<>();
+        this.uniqueRecords = new ArrayList<>();
+        this.duplicatedRecords = new ArrayList<>();
         this.mailChimpClient = new MailChimpClient(task);
     }
 
@@ -88,20 +95,16 @@ public class MailChimpRecordBuffer
 
             records.add(record);
             if (requestCount >= task.getMaxRecordsPerRequest()) {
-                ObjectNode subcribers = processSubcribers(records, task);
-                ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
+                filterDuplicatedRecords();
+                pushData();
 
                 if (totalCount % 1000 == 0) {
                     LOG.info("Pushed {} records", totalCount);
                 }
 
-                LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
-                         reportResponse.getTotalCreated(),
-                         reportResponse.getTotalUpdated(),
-                         reportResponse.getErrorCount());
-                mailChimpClient.handleErrors(reportResponse.getErrors());
-
                 records = new ArrayList<>();
+                uniqueRecords = new ArrayList<>();
+                duplicatedRecords = new ArrayList<>();
                 requestCount = 0;
             }
         }
@@ -121,14 +124,11 @@ public class MailChimpRecordBuffer
     {
         try {
             if (records.size() > 0) {
-                ObjectNode subcribers = processSubcribers(records, task);
-                ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
-                LOG.info("Pushed {} records", records.size());
-                LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
-                         reportResponse.getTotalCreated(),
-                         reportResponse.getTotalUpdated(),
-                         reportResponse.getErrorCount());
-                mailChimpClient.handleErrors(reportResponse.getErrors());
+                filterDuplicatedRecords();
+                pushData();
+                records = new ArrayList<>();
+                uniqueRecords = new ArrayList<>();
+                duplicatedRecords = new ArrayList<>();
             }
 
             return Exec.newTaskReport().set("pushed", totalCount);
@@ -163,8 +163,6 @@ public class MailChimpRecordBuffer
     private ObjectNode processSubcribers(final List<JsonNode> data, final PluginTask task)
             throws JsonProcessingException
     {
-        LOG.info("Start to process subscriber data");
-
         // Should loop the names and get the id of interest categories.
         // The reason why we put categories validation here because we can not share data between instance.
         categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
@@ -277,4 +275,46 @@ public class MailChimpRecordBuffer
 
         return interests;
     }
+
+    private void filterDuplicatedRecords()
+    {
+        Set<String> uniqueEmails = new HashSet<>();
+        for (JsonNode node : records) {
+            if (uniqueEmails.contains(node.findPath(task.getEmailColumn()).asText())) {
+                duplicatedRecords.add(node);
+            }
+            else {
+                uniqueEmails.add(node.findPath(task.getEmailColumn()).asText());
+                uniqueRecords.add(node);
+            }
+        }
+    }
+
+    private void pushData() throws JsonProcessingException
+    {
+        long startTime = System.currentTimeMillis();
+        ObjectNode subscribers = processSubcribers(uniqueRecords, task);
+        ReportResponse reportResponse = mailChimpClient.push(subscribers, task);
+
+        LOG.info("Done with {} record(s). Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
+                 records.size(), reportResponse.getTotalCreated(),
+                 reportResponse.getTotalUpdated(),
+                 reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
+        mailChimpClient.handleErrors(reportResponse.getErrors());
+
+        if (duplicatedRecords.size() > 0) {
+            LOG.info("Start to process {} duplicated record(s)", duplicatedRecords.size());
+            for (JsonNode duplicatedRecord : duplicatedRecords) {
+                startTime = System.currentTimeMillis();
+                subscribers = processSubcribers(Arrays.asList(duplicatedRecord), task);
+                reportResponse = mailChimpClient.push(subscribers, task);
+
+                LOG.info("Done. Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
+                         reportResponse.getTotalCreated(),
+                         reportResponse.getTotalUpdated(),
+                         reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
+                mailChimpClient.handleErrors(reportResponse.getErrors());
+            }
+        }
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java

@@ -12,6 +12,9 @@ public class InterestCategoriesResponse
     @JsonProperty("categories")
     private List<CategoriesResponse> categories;
 
+    @JsonProperty("total_items")
+    private int totalItems;
+
     public List<CategoriesResponse> getCategories()
     {
         return categories;
@@ -21,4 +24,14 @@ public class InterestCategoriesResponse
     {
         this.categories = categories;
     }
+
+    public int getTotalItems()
+    {
+        return totalItems;
+    }
+
+    public void setTotalItems(int totalItems)
+    {
+        this.totalItems = totalItems;
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/model/MergeFields.java

@@ -12,6 +12,9 @@ public class MergeFields
     @JsonProperty("merge_fields")
     private List<MergeField> mergeFields;
 
+    @JsonProperty("total_items")
+    private int totalItems;
+
     public List<MergeField> getMergeFields()
     {
         return mergeFields;
@@ -21,4 +24,14 @@ public class MergeFields
     {
         this.mergeFields = mergeFields;
     }
+
+    public int getTotalItems()
+    {
+        return totalItems;
+    }
+
+    public void setTotalItems(int totalItems)
+    {
+        this.totalItems = totalItems;
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.19
+  version: 0.3.20
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-10 00:00:00.000000000 Z
+date: 2017-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -89,7 +89,7 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/jetty-io-9.2.14.v20151106.jar
-- classpath/embulk-output-mailchimp-0.3.19.jar
+- classpath/embulk-output-mailchimp-0.3.20.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar