embulk-output-mailchimp 0.3.20 → 0.3.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c279827f48671c120b7574d8de2c05669014e4fe
-  data.tar.gz: 96c677aa514f29b5588bc0d901383cda9a84b7c4
+  metadata.gz: 504301e8e2482b1d046fb9a880ab63a631dac2ca
+  data.tar.gz: 053d271506d3a0dcb9020466ed703ccfe4d17469
 SHA512:
-  metadata.gz: ea72fabb9264611f2146d3c64e540f7179ce76298701b29bb830417fb25dfede981353c433fc8731cc81c1b0415aa6ad8dda18bc3cab30602419f7001237b508
-  data.tar.gz: 1c939eb9fd3558cefa4234d07961b7c25e5aa943f1f6cf59519e8191a7abda2376d72f3453e9d71f0ff79ac7baae27af7ee87c4bac434d2ddd850e17d1c309a4
+  metadata.gz: c08f3cfe6e2de7f6c2c35b04dac4f50b99ab1457c285e7ba42d8c6aed854d5922e640b3e1c089039f8453048b22996bd52e0a1fb689e9b7af8cab4e5fd50954c
+  data.tar.gz: d5dca1a6c14f1c992fe0f3e34a987052e6aa0ee8682bfa6114c0d510fcb790e5945e9d8e46091b7184afab7279143f08fb7fb5f9690724d612d9ec803f0908ea

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.21 - 2018-03-02
+- Refactor code to improve performance and fix NPE with merge fields that contain case sensitive [#40](https://github.com/treasure-data/embulk-output-mailchimp/pull/40)
+
 ## 0.3.20 - 2017-10-16
 - Added pagination for interest categories [#39](https://github.com/treasure-data/embulk-output-mailchimp/pull/39)
 - Refactor check list id to avoid confusing when get 404 error [#38](https://github.com/treasure-data/embulk-output-mailchimp/pull/38)

data/README.md

@@ -26,6 +26,7 @@ add e-mail to List in MailChimp.
 - **language_column**: column name for language (string, optional, default: nil)
 - **double_optin**: control whether to send an opt-in confirmation email (boolean, default: true)
 - **max_records_per_request**: The max records per batch request. MailChimp API enables max records is 500 per batch request (int, default: 500)
+- **sleep_between_requests_millis**: The time to sleep between requests to avoid flood MailChimp API (int, default: 3000)
 
 ## Example
 
@@ -51,5 +52,5 @@ out:
 ## Build
 
 ```
-$ rake
+$ ./gradlew gem
 ```

data/build.gradle

@@ -18,22 +18,21 @@ configurations {
     provided
 }
 
-version = "0.3.20"
+version = "0.3.21"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.15"
-    provided "org.embulk:embulk-core:0.8.15"
+    compile  "org.embulk:embulk-core:0.8.25"
+    provided "org.embulk:embulk-core:0.8.25"
     compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
     compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.15:tests"
-    testCompile "org.embulk:embulk-test:0.8.15"
-    testCompile "org.embulk:embulk-standards:0.8.15"
+    testCompile "org.embulk:embulk-core:0.8.25:tests"
+    testCompile "org.embulk:embulk-test:0.8.25"
+    testCompile "org.embulk:embulk-standards:0.8.25"
     testCompile "org.mockito:mockito-core:2.+"
 }
 

data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java

@@ -5,18 +5,17 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.MissingNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
-import com.google.common.base.Joiner;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import org.eclipse.jetty.client.HttpResponseException;
-import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.base.restclient.jackson.StringJsonParser;
 import org.embulk.config.ConfigException;
 import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
 import org.embulk.output.mailchimp.helper.MailChimpHelper;
+import org.embulk.output.mailchimp.helper.MailChimpRetryable;
 import org.embulk.output.mailchimp.model.CategoriesResponse;
 import org.embulk.output.mailchimp.model.ErrorResponse;
 import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
@@ -24,7 +23,6 @@ import org.embulk.output.mailchimp.model.InterestResponse;
 import org.embulk.output.mailchimp.model.InterestsResponse;
 import org.embulk.output.mailchimp.model.MergeField;
 import org.embulk.output.mailchimp.model.MergeFields;
-import org.embulk.output.mailchimp.model.MetaDataResponse;
 import org.embulk.output.mailchimp.model.ReportResponse;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
@@ -38,19 +36,16 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.embulk.output.mailchimp.model.AuthMethod.API_KEY;
-import static org.embulk.output.mailchimp.model.AuthMethod.OAUTH;
-
 /**
  * Created by thangnc on 4/25/17.
  */
 public class MailChimpClient
 {
     private static final Logger LOG = Exec.getLogger(MailChimpClient.class);
-    private static final String API_VERSION = "3.0";
-    private static String mailchimpEndpoint;
-    private MailChimpHttpClient client;
-    private final ObjectMapper mapper;
+    private final ObjectMapper mapper = new ObjectMapper()
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+            .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+    private StringJsonParser jsonParser = new StringJsonParser();
 
     /**
      * Instantiates a new Mail chimp client.
@@ -59,12 +54,6 @@ public class MailChimpClient
      */
     public MailChimpClient(final PluginTask task)
     {
-        mailchimpEndpoint = Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION);
-        this.client = new MailChimpHttpClient(task);
-        this.mapper = new ObjectMapper()
-                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
-        extractDataCenter(task);
         findList(task);
     }
 
@@ -75,24 +64,19 @@ public class MailChimpClient
      * @param node the data
      * @param task the task
      * @return the report response
-     * @throws JsonProcessingException the json processing exception
      */
-    ReportResponse push(final ObjectNode node, PluginTask task)
-            throws JsonProcessingException
+    public ReportResponse push(final ObjectNode node, PluginTask task) throws JsonProcessingException
     {
-        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
-                                               task.getListId());
-
-        JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
-        client.avoidFlushAPI("Process next request");
+        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
+            String response = retryable.post(MessageFormat.format("/lists/{0}", task.getListId()),
+                                             "application/json;utf-8",
+                                             node.toString());
+            if (response != null && !response.isEmpty()) {
+                return mapper.treeToValue(jsonParser.parseJsonObject(response), ReportResponse.class);
+            }
 
-        if (response instanceof MissingNode) {
-            ReportResponse reportResponse = new ReportResponse();
-            reportResponse.setErrors(new ArrayList<ErrorResponse>());
-            return reportResponse;
+            throw new DataException("The json data in response were broken.");
         }
-
-        return mapper.treeToValue(response, ReportResponse.class);
     }
 
     /**
@@ -100,7 +84,7 @@ public class MailChimpClient
      *
      * @param errorResponses the error responses
      */
-    void handleErrors(List<ErrorResponse> errorResponses)
+    public void handleErrors(List<ErrorResponse> errorResponses)
     {
         if (!errorResponses.isEmpty()) {
             StringBuilder errorMessage = new StringBuilder();
@@ -125,76 +109,77 @@ public class MailChimpClient
      * @return the map
      * @throws JsonProcessingException the json processing exception
      */
-    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
+    public Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
             throws JsonProcessingException
     {
-        Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
-        if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
-            List<String> interestCategoryNames = task.getGroupingColumns().get();
-
-            int count = 100;
-            int offset = 0;
-            int page = 1;
-            boolean hasMore = true;
-            JsonNode response;
-            List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
-
-            while (hasMore) {
-                String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories?count={1}&offset={2}",
+        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
+            Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
+            if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
+                List<String> interestCategoryNames = task.getGroupingColumns().get();
+
+                int count = 100;
+                int offset = 0;
+                int page = 1;
+                boolean hasMore = true;
+                JsonNode response;
+                List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
+
+                while (hasMore) {
+                    String path = MessageFormat.format("/lists/{0}/interest-categories?count={1}&offset={2}",
                                                        task.getListId(),
                                                        count,
                                                        offset);
-
-                response = client.sendRequest(endpoint, HttpMethod.GET, task);
-                InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
-                                                                                           InterestCategoriesResponse.class);
-
-                allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
-                if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
-                    offset = count;
-                    page++;
-                }
-                else {
-                    hasMore = false;
+                    response = jsonParser.parseJsonObject(retryable.get(path));
+                    InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
+                                                                                               InterestCategoriesResponse.class);
+
+                    allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
+                    if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
+                        offset = count;
+                        page++;
+                    }
+                    else {
+                        hasMore = false;
+                    }
                 }
-            }
 
-            Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
-            {
-                @Override
-                public String apply(CategoriesResponse input)
+                Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
                 {
-                    return input.getTitle().toLowerCase();
+                    @Override
+                    public String apply(CategoriesResponse input)
+                    {
+                        return input.getTitle().toLowerCase();
+                    }
+                };
+
+                // Transform to a list of available category names and validate with data that user input
+                ImmutableList<String> availableCategories = FluentIterable
+                        .from(allCategoriesResponse)
+                        .transform(function)
+                        .toList();
+
+                for (String category : interestCategoryNames) {
+                    if (!availableCategories.contains(category)) {
+                        throw new ConfigException("Invalid interest category name: '" + category + "'");
+                    }
                 }
-            };
-
-            // Transform to a list of available category names and validate with data that user input
-            ImmutableList<String> availableCategories = FluentIterable
-                    .from(allCategoriesResponse)
-                    .transform(function)
-                    .toList();
 
-            for (String category : interestCategoryNames) {
-                if (!availableCategories.contains(category)) {
-                    throw new ConfigException("Invalid interest category name: '" + category + "'");
-                }
-            }
-
-            for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
-                String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
+                for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
+                    String detailPath = MessageFormat.format("/lists/{0}/interest-categories/{1}/interests",
                                                              task.getListId(),
                                                              categoriesResponse.getId());
-                response = client.sendRequest(detailEndpoint, HttpMethod.GET, task);
+                    response = jsonParser.parseJsonObject(retryable.get(detailPath));
 
-                // Avoid flush MailChimp API
-                client.avoidFlushAPI("Fetching next category's interests");
-                InterestsResponse interestsResponse = mapper.treeToValue(response, InterestsResponse.class);
-                categories.put(categoriesResponse.getTitle().toLowerCase(),
-                               convertInterestCategoryToMap(interestsResponse.getInterests()));
+                    // Avoid flood MailChimp API
+                    avoidFloodAPI("Fetching next category's interests", task.getSleepBetweenRequestsMillis());
+                    InterestsResponse interestsResponse = mapper.treeToValue(response, InterestsResponse.class);
+                    categories.put(categoriesResponse.getTitle().toLowerCase(),
+                                   convertInterestCategoryToMap(interestsResponse.getInterests()));
+                }
             }
-        }
 
-        return categories;
+            return categories;
+        }
     }
 
     /**
@@ -205,76 +190,45 @@ public class MailChimpClient
      * @return the map
      * @throws JsonProcessingException the json processing exception
      */
-    Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
+    public Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
     {
-        int count = 100;
-        int offset = 0;
-        int page = 1;
-        boolean hasMore = true;
-        List<MergeField> allMergeFields = new ArrayList<>();
-
-        while (hasMore) {
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields?count={1}&offset={2}",
+        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
+            int count = 100;
+            int offset = 0;
+            int page = 1;
+            boolean hasMore = true;
+            List<MergeField> allMergeFields = new ArrayList<>();
+
+            while (hasMore) {
+                String path = MessageFormat.format("/lists/{0}/merge-fields?count={1}&offset={2}",
                                                    task.getListId(),
                                                    count,
                                                    offset);
 
-            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-            MergeFields mergeFields = mapper.treeToValue(response,
-                                                         MergeFields.class);
+                JsonNode response = jsonParser.parseJsonObject(retryable.get(path));
+                MergeFields mergeFields = mapper.treeToValue(response,
+                                                             MergeFields.class);
 
-            allMergeFields.addAll(mergeFields.getMergeFields());
+                allMergeFields.addAll(mergeFields.getMergeFields());
 
-            if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
-                offset = count;
-                page++;
-            }
-            else {
-                hasMore = false;
-            }
-        }
-
-        return convertMergeFieldToMap(allMergeFields);
-    }
-
-    private void extractDataCenter(final PluginTask task)
-    {
-        if (task.getAuthMethod() == OAUTH) {
-            // Extract data center from meta data URL
-            JsonNode response = client.sendRequest("https://login.mailchimp.com/oauth2/metadata", HttpMethod.GET, task);
-            MetaDataResponse metaDataResponse;
-            try {
-                metaDataResponse = mapper.treeToValue(response, MetaDataResponse.class);
-                mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, metaDataResponse.getDc());
-            }
-            catch (JsonProcessingException e) {
-                throw new DataException(e);
-            }
-        }
-        else if (task.getAuthMethod() == API_KEY && task.getApikey().isPresent()) {
-            // Authenticate and return data center
-            if (!task.getApikey().get().contains("-")) {
-                throw new ConfigException("API Key format invalid.");
+                if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
+                    offset = count;
+                    page++;
+                }
+                else {
+                    hasMore = false;
+                }
             }
 
-            String domain = task.getApikey().get().split("-")[1];
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/", domain);
-            try {
-                client.sendRequest(endpoint, HttpMethod.GET, task);
-                mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, domain);
-            }
-            catch (HttpResponseException re) {
-                throw new ConfigException("Your API key may be invalid, or you've attempted to access the wrong datacenter.");
-            }
+            return convertMergeFieldToMap(allMergeFields);
         }
     }
 
     private void findList(final PluginTask task)
     {
-        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
-                                               task.getListId());
-        try {
-            client.sendRequest(endpoint, HttpMethod.GET, task);
+        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
+            jsonParser.parseJsonObject(retryable.get(MessageFormat.format("/lists/{0}",
+                                                                          task.getListId())));
         }
         catch (HttpResponseException hre) {
             throw new ConfigException("The `list id` could not be found.");
@@ -319,4 +273,15 @@ public class MailChimpClient
         int totalPage = count / pageSize + (count % pageSize > 0 ? 1 : 0);
         return page < totalPage;
     }
+
+    public void avoidFloodAPI(final String reason, final long millis)
+    {
+        try {
+            LOG.info("{} in {}ms...", reason, millis);
+            Thread.sleep(millis);
+        }
+        catch (InterruptedException e) {
+            LOG.warn("Failed to sleep: {}", e.getMessage());
+        }
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java

@@ -107,6 +107,10 @@ public class MailChimpOutputPluginDelegate
         @Config("max_records_per_request")
         @ConfigDefault("500")
         int getMaxRecordsPerRequest();
+
+        @Config("sleep_between_requests_millis")
+        @ConfigDefault("3000")
+        int getSleepBetweenRequestsMillis();
     }
 
     /**
@@ -129,6 +133,10 @@ public class MailChimpOutputPluginDelegate
             if (!task.getApikey().isPresent() || isNullOrEmpty(task.getApikey().get())) {
                 throw new ConfigException("'apikey' is required when auth_method is 'api_key'");
             }
+
+            if (!task.getApikey().get().contains("-")) {
+                throw new ConfigException("apikey's format invalid.");
+            }
         }
 
         if (isNullOrEmpty(task.getListId())) {

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -150,7 +150,7 @@ public class MailChimpRecordBuffer
     @Override
     public void close()
     {
-        // Do not implement
+//      // Do not close here
     }
 
     /**
@@ -165,10 +165,14 @@ public class MailChimpRecordBuffer
     {
         // Should loop the names and get the id of interest categories.
         // The reason why we put categories validation here because we can not share data between instance.
-        categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
+        if (categories == null) {
+            categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
+        }
 
         // Extract merge fields detail
-        availableMergeFields = mailChimpClient.extractMergeFieldsFromList(task);
+        if (availableMergeFields == null) {
+            availableMergeFields = mailChimpClient.extractMergeFieldsFromList(task);
+        }
 
         // Required merge fields
         Map<String, String> map = new HashMap<>();
@@ -208,7 +212,7 @@ public class MailChimpRecordBuffer
                             String value = input.hasNonNull(column.getName()) ? input.findValue(column.getName()).asText() : "";
 
                             // Try to convert to Json from string with the merge field's type is address
-                            if (availableMergeFields.get(column.getName()).getType()
+                            if (availableMergeFields.get(column.getName().toLowerCase()).getType()
                                     .equals(MergeField.MergeFieldType.ADDRESS.getType())) {
                                 JsonNode addressNode = toJsonNode(value);
                                 if (addressNode instanceof NullNode) {
@@ -243,31 +247,32 @@ public class MailChimpRecordBuffer
         };
     }
 
-    private ObjectNode buildInterestCategories(final MailChimpOutputPluginDelegate.PluginTask task,
-                                               final JsonNode input)
+    private ObjectNode buildInterestCategories(final PluginTask task, final JsonNode input)
     {
         ObjectNode interests = JsonNodeFactory.instance.objectNode();
 
-        for (String category : task.getGroupingColumns().get()) {
-            String inputValue = input.findValue(category).asText();
-            List<String> interestValues = fromCommaSeparatedString(inputValue);
-            Map<String, InterestResponse> availableCategories = categories.get(category);
-
-            // Only update user-predefined categories if replace interests != true
-            if (!task.getReplaceInterests()) {
-                for (String interestValue : interestValues) {
-                    if (availableCategories.get(interestValue) != null) {
-                        interests.put(availableCategories.get(interestValue).getId(), true);
-                    }
-                }
-            } // Otherwise, force update all categories include user-predefined categories
-            else if (task.getReplaceInterests()) {
-                for (String availableCategory : availableCategories.keySet()) {
-                    if (interestValues.contains(availableCategory)) {
-                        interests.put(availableCategories.get(availableCategory).getId(), true);
+        if (task.getGroupingColumns().isPresent()) {
+            for (String category : task.getGroupingColumns().get()) {
+                String inputValue = input.findValue(category).asText();
+                List<String> interestValues = fromCommaSeparatedString(inputValue);
+                Map<String, InterestResponse> availableCategories = categories.get(category);
+
+                // Only update user-predefined categories if replace interests != true
+                if (!task.getReplaceInterests()) {
+                    for (String interestValue : interestValues) {
+                        if (availableCategories.get(interestValue) != null) {
+                            interests.put(availableCategories.get(interestValue).getId(), true);
+                        }
                     }
-                    else {
-                        interests.put(availableCategories.get(availableCategory).getId(), false);
+                } // Otherwise, force update all categories include user-predefined categories
+                else if (task.getReplaceInterests()) {
+                    for (String availableCategory : availableCategories.keySet()) {
+                        if (interestValues.contains(availableCategory)) {
+                            interests.put(availableCategories.get(availableCategory).getId(), true);
+                        }
+                        else {
+                            interests.put(availableCategories.get(availableCategory).getId(), false);
+                        }
                     }
                 }
             }
@@ -302,6 +307,8 @@ public class MailChimpRecordBuffer
                  reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
         mailChimpClient.handleErrors(reportResponse.getErrors());
 
+        mailChimpClient.avoidFloodAPI("Process next request", task.getSleepBetweenRequestsMillis());
+
         if (duplicatedRecords.size() > 0) {
             LOG.info("Start to process {} duplicated record(s)", duplicatedRecords.size());
             for (JsonNode duplicatedRecord : duplicatedRecords) {

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java

@@ -0,0 +1,224 @@
+package org.embulk.output.mailchimp.helper;
+
+import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpResponseException;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.embulk.base.restclient.jackson.StringJsonParser;
+import org.embulk.config.ConfigException;
+import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.util.retryhelper.jetty92.DefaultJetty92ClientCreator;
+import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
+import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
+import org.slf4j.Logger;
+
+import java.text.MessageFormat;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import static org.eclipse.jetty.http.HttpHeader.AUTHORIZATION;
+import static org.eclipse.jetty.http.HttpMethod.GET;
+import static org.eclipse.jetty.http.HttpMethod.POST;
+import static org.embulk.output.mailchimp.model.AuthMethod.API_KEY;
+import static org.embulk.output.mailchimp.model.AuthMethod.OAUTH;
+
+public class MailChimpRetryable implements AutoCloseable
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpRetryable.class);
+    private static final int READER_TIMEOUT_MILLIS = 300000;
+    private static final String API_VERSION = "3.0";
+    private final Jetty92RetryHelper retryHelper;
+    private final PluginTask pluginTask;
+    private static TokenHolder tokenHolder;
+    protected StringJsonParser jsonParser = new StringJsonParser();
+    private String authorizationHeader;
+
+    public MailChimpRetryable(final PluginTask pluginTask)
+    {
+        this.retryHelper = new Jetty92RetryHelper(pluginTask.getMaximumRetries(),
+                                                  pluginTask.getInitialRetryIntervalMillis(),
+                                                  pluginTask.getMaximumRetryIntervalMillis(),
+                                                  new DefaultJetty92ClientCreator(pluginTask.getTimeoutMillis(),
+                                                                                  pluginTask.getTimeoutMillis()));
+        this.pluginTask = pluginTask;
+        authorizationHeader = buildAuthorizationHeader(pluginTask);
+    }
+
+    public String get(final String path)
+    {
+        return sendRequest(path, null);
+    }
+
+    public String post(final String path, String contentType, String body)
+    {
+        return sendRequest(path, new StringContentProvider(contentType, body, Charsets.UTF_8));
+    }
+
+    private String sendRequest(final String path, final StringContentProvider contentProvider)
+    {
+        return retryHelper.requestWithRetry(
+                new StringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
+                new Jetty92SingleRequester()
+                {
+                    @Override
+                    public void requestOnce(HttpClient client, Response.Listener responseListener)
+                    {
+                        createTokenHolder(client);
+                        Request request = client.newRequest(tokenHolder.getEndpoint() + path)
+                                .header(AUTHORIZATION, authorizationHeader)
+                                .method(GET);
+                        if (contentProvider != null) {
+                            request = request.method(POST).content(contentProvider);
+                        }
+                        request.send(responseListener);
+                    }
+
+                    @Override
+                    protected boolean isResponseStatusToRetry(Response response)
+                    {
+                        // Retry if it's a server or rate limit exceeded error
+                        return (response.getStatus() != 500 && response.getStatus() / 100 != 4) || response.getStatus() == 429;
+                    }
+
+                    @Override
+                    protected boolean isExceptionToRetry(Exception exception)
+                    {
+                        // This check is to make sure if the original exception is retryable, i.e.
+                        // server not found, internal server error...
+                        if (exception instanceof ConfigException || exception instanceof ExecutionException) {
+                            return toRetry((Exception) exception.getCause());
+                        }
+                        return exception instanceof TimeoutException || super.isExceptionToRetry(exception);
+                    }
+                });
+    }
+
+    @Override
+    public void close()
+    {
+        if (retryHelper != null) {
+            retryHelper.close();
+        }
+    }
+
+    /**
+     * MailChimp API v3 supports non expires access_token. Then no need refresh_token
+     *
+     * @param task
+     * @return
+     */
+    private String buildAuthorizationHeader(final PluginTask task)
+    {
+        switch (task.getAuthMethod()) {
+            case OAUTH:
+                return "OAuth " + task.getAccessToken().orNull();
+            case API_KEY:
+                return "Basic " + Base64Variants.MIME_NO_LINEFEEDS
+                        .encode(("apikey" + ":" + task.getApikey().orNull()).getBytes());
+            default:
+                throw new ConfigException("Not supported method");
+        }
+    }
+
+    private TokenHolder createTokenHolder(final HttpClient client)
+    {
+        if (tokenHolder != null) {
+            return tokenHolder;
+        }
+
+        LOG.info("Create new token holder and extract data center");
+
+        if (pluginTask.getAuthMethod() == OAUTH) {
+            try {
+                // Extract data center from meta data URL
+                ContentResponse contentResponse = client.newRequest("https://login.mailchimp.com/oauth2/metadata")
+                        .method(GET)
+                        .header("Authorization", authorizationHeader)
+                        .send();
+
+                if (contentResponse.getStatus() == 200) {
+                    ObjectNode objectNode = jsonParser.parseJsonObject(contentResponse.getContentAsString());
+                    String endpoint = MessageFormat.format(Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION),
+                                                           objectNode.get("dc").asText());
+                    tokenHolder = new TokenHolder(pluginTask.getAccessToken().orNull(), null, endpoint);
+                    return tokenHolder;
+                }
+
+                String message = String.format("%s %d %s",
+                                               contentResponse.getVersion(),
+                                               contentResponse.getStatus(),
+                                               contentResponse.getReason());
+                throw new HttpResponseException(message, contentResponse);
+            }
+            catch (Exception ex) {
+                throw new ConfigException("Unable to connect the data center", ex);
+            }
+        }
+        else if (pluginTask.getAuthMethod() == API_KEY) {
+            try {
+                // Authenticate and return data center
+                String domain = pluginTask.getApikey().get().split("-")[1];
+                String endpoint = MessageFormat.format(Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION),
+                                                       domain);
+                ContentResponse contentResponse = client.newRequest(endpoint + "/")
+                        .method(GET)
+                        .header("Authorization", "Basic " + Base64Variants.MIME_NO_LINEFEEDS
+                                .encode(("apikey" + ":" + pluginTask.getApikey().get()).getBytes()))
+                        .send();
+
+                if (contentResponse.getStatus() == 200) {
+                    tokenHolder = new TokenHolder(null, pluginTask.getApikey().orNull(), endpoint);
+                    return tokenHolder;
+                }
+
+                String message = String.format("%s %d %s",
+                                               contentResponse.getVersion(),
+                                               contentResponse.getStatus(),
+                                               contentResponse.getReason());
+                throw new HttpResponseException(message, contentResponse);
+            }
+            catch (Exception ex) {
+                throw new ConfigException("Your API key may be invalid, or you've attempted to access the wrong datacenter.");
+            }
+        }
+
+        throw new ConfigException("Not supported auth method");
+    }
+}
+
+class TokenHolder
+{
+    private String accessToken;
+    private String apiKey;
+    private String endpoint;
+
+    public TokenHolder(final String accessToken, final String apiKey, final String endpoint)
+    {
+        this.accessToken = accessToken;
+        this.apiKey = apiKey;
+        this.endpoint = endpoint;
+    }
+
+    public String getAccessToken()
+    {
+        return accessToken;
+    }
+
+    public String getApiKey()
+    {
+        return apiKey;
+    }
+
+    public String getEndpoint()
+    {
+        return endpoint;
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.20
+  version: 0.3.21
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-16 00:00:00.000000000 Z
+date: 2018-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -62,11 +62,11 @@ files:
 - gradlew.bat
 - lib/embulk/output/mailchimp.rb
 - src/main/java/org/embulk/output/mailchimp/MailChimpClient.java
-- src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpOutputPlugin.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java
 - src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java
+- src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java
 - src/main/java/org/embulk/output/mailchimp/model/AddressMergeFieldAttribute.java
 - src/main/java/org/embulk/output/mailchimp/model/AuthMethod.java
 - src/main/java/org/embulk/output/mailchimp/model/CategoriesResponse.java
@@ -89,7 +89,7 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/jetty-io-9.2.14.v20151106.jar
-- classpath/embulk-output-mailchimp-0.3.20.jar
+- classpath/embulk-output-mailchimp-0.3.21.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar

data/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java

@@ -1,200 +0,0 @@
-package org.embulk.output.mailchimp;
-
-import com.fasterxml.jackson.core.Base64Variants;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.MissingNode;
-import com.google.common.base.Throwables;
-import org.eclipse.jetty.client.HttpClient;
-import org.eclipse.jetty.client.api.Request;
-import org.eclipse.jetty.client.api.Response;
-import org.eclipse.jetty.client.util.StringContentProvider;
-import org.eclipse.jetty.http.HttpMethod;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.embulk.config.ConfigException;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
-import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
-import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
-import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Created by thangnc on 4/14/17.
- */
-public class MailChimpHttpClient
-{
-    private static final Logger LOG = Exec.getLogger(MailChimpHttpClient.class);
-    private final ObjectMapper jsonMapper = new ObjectMapper()
-            .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false)
-            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
-    /**
-     * Instantiates a new Mailchimp http client.
-     *
-     * @param task the task
-     */
-    public MailChimpHttpClient(MailChimpOutputPluginDelegate.PluginTask task)
-    {
-    }
-
-    public JsonNode sendRequest(final String endpoint, final HttpMethod method,
-                                final MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        return sendRequest(endpoint, method, "", task);
-    }
-
-    public JsonNode sendRequest(final String endpoint, final HttpMethod method, final String content,
-                                final MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        try (final Jetty92RetryHelper retryHelper = createRetryHelper(task)) {
-            final String authorizationHeader = getAuthorizationHeader(task);
-
-            String responseBody = retryHelper.requestWithRetry(
-                    new StringJetty92ResponseEntityReader(task.getTimeoutMillis()),
-                    new Jetty92SingleRequester()
-                    {
-                        @Override
-                        public void requestOnce(HttpClient client, Response.Listener responseListener)
-                        {
-                            Request request = client
-                                    .newRequest(endpoint)
-                                    .timeout(task.getTimeoutMillis(), TimeUnit.MILLISECONDS)
-                                    .accept("application/json")
-                                    .method(method);
-                            if (method == HttpMethod.POST || method == HttpMethod.PUT) {
-                                request.content(new StringContentProvider(content), "application/json;utf-8");
-                            }
-
-                            if (!authorizationHeader.isEmpty()) {
-                                request.header("Authorization", authorizationHeader);
-                            }
-                            request.send(responseListener);
-                        }
-
-                        @Override
-                        public boolean isResponseStatusToRetry(Response response)
-                        {
-                            int status = response.getStatus();
-
-                            return status == 429 || status / 100 != 4;
-                        }
-
-                        @Override
-                        protected boolean isExceptionToRetry(Exception exception)
-                        {
-                            LOG.error("Exception to retry.", exception);
-                            // This check is to make sure the exception is retryable, e.g: server not found, internal server error...
-                            if (exception instanceof ConfigException || exception instanceof ExecutionException) {
-                                return toRetry((Exception) exception.getCause());
-                            }
-
-                            return exception instanceof TimeoutException || super.isExceptionToRetry(exception);
-                        }
-                    });
-
-            return responseBody != null && !responseBody.isEmpty() ? parseJson(responseBody) : MissingNode.getInstance();
-        }
-        catch (Exception ex) {
-            LOG.error("Exception occurred while sending request.", ex);
-            throw Throwables.propagate(ex);
-        }
-    }
-
-    private JsonNode parseJson(final String json)
-            throws DataException
-    {
-        try {
-            return this.jsonMapper.readTree(json);
-        }
-        catch (IOException ex) {
-            // Try to parse invalid json before throwing exception
-            return parseInvalidJsonString(json);
-        }
-    }
-
-    // Sometimes, the MailChimp API returns invalid JSON when we pushed a large of data. ObjectMapper can not read string json.
-    // So we have to use this method to parse string and build a json string as ReportResponse
-    // E.g invalid json response from MailChimp https://gist.github.com/thangnc/dc94026e4b13b728b7303f402b458b05
-    private JsonNode parseInvalidJsonString(final String json)
-    {
-        int totalCreatedIndex = json.indexOf("\"total_created\"");
-        int totalUpdatedIndex = json.indexOf("\"total_updated\"");
-        int errorCountIndex = json.indexOf("\"error_count\"");
-        int errorsIndex = json.indexOf("\"errors\"");
-
-        StringBuilder validJson = new StringBuilder();
-        validJson.append("{").append(json.substring(errorsIndex, totalCreatedIndex - 1)).append(",");
-        validJson.append(json.substring(totalCreatedIndex, totalCreatedIndex + "\"total_created\"".length() + 2)).append(",");
-        validJson.append(json.substring(totalUpdatedIndex, totalUpdatedIndex + "\"total_updated\"".length() + 2)).append(",");
-        validJson.append(json.substring(errorCountIndex, errorCountIndex + "\"error_count\"".length() + 2)).append("}");
-
-        try {
-            return this.jsonMapper.readTree(validJson.toString());
-        }
-        catch (IOException ex) {
-            throw new DataException(ex);
-        }
-    }
-
-    public void avoidFlushAPI(String reason)
-    {
-        try {
-            LOG.info("{} in 5s...", reason);
-            Thread.sleep(5000);
-        }
-        catch (InterruptedException e) {
-            LOG.warn("Failed to sleep: {}", e.getMessage());
-        }
-    }
-
-    /**
-     * MailChimp API v3 supports non expires access_token. Then no need refresh_token
-     *
-     * @param task
-     * @return
-     */
-    private String getAuthorizationHeader(final MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        switch (task.getAuthMethod()) {
-            case OAUTH:
-                return "OAuth " + task.getAccessToken().orNull();
-            case API_KEY:
-                return "Basic " + Base64Variants.MIME_NO_LINEFEEDS
-                        .encode(("apikey" + ":" + task.getApikey().orNull()).getBytes());
-            default:
-                throw new ConfigException("Not supported method");
-        }
-    }
-
-    private Jetty92RetryHelper createRetryHelper(MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        return new Jetty92RetryHelper(
-                task.getMaximumRetries(),
-                task.getInitialRetryIntervalMillis(),
-                task.getMaximumRetryIntervalMillis(),
-                new Jetty92ClientCreator()
-                {
-                    @Override
-                    public HttpClient createAndStart()
-                    {
-                        HttpClient client = new HttpClient(new SslContextFactory());
-                        try {
-                            client.start();
-                            return client;
-                        }
-                        catch (Exception e) {
-                            throw Throwables.propagate(e);
-                        }
-                    }
-                });
-    }
-}