embulk-output-mailchimp 0.3.28 → 0.3.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b429373deb2a79bd444d223f70818c7a7517bd63
-  data.tar.gz: 06fb08e83602804217e386754e39cd7cb542c2e8
+  metadata.gz: 37dfc8c615f96c0c1652b421bb1b716fbf44e578
+  data.tar.gz: 563dfbcf427399d4d78b4f1fd277d5eb8a437835
 SHA512:
-  metadata.gz: b096372c93fcb6a0c2930d0b98ade33a1d52277f65fd1554e6d80178ee2179b300847c69c907d4e2cbb1279f7028e6eac3804d123f8e92681e1955182df5bc22
-  data.tar.gz: efd31474670848d2457b2fdc5ea68b990902971543e884dd473fcb57064b34687a79beb95188bc06349e5448cac1a4919b91cadc90083a0fe39cca32d09c5c60
+  metadata.gz: 95ecae408533dbea06c19215d01e6a77842f5ee23a5948768352fed63470fea2ae486fc3c037c2a4bda6fe20b0d71144bc0eb087f7643768c80c0057ab59a758
+  data.tar.gz: c577620282aeb24d5c3c4bed48ded23338ae4f271637bd6b247a39c536c3a83ef7b3168db49fc734f15bdd879c1cc3ea0ec76dd04addd5b0e377170d8f528099

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.3.31 - 2019-09-30
+
+- Fix: retries always fail
+- Fix: correct pagination
+
+## 0.3.30 - 2019-01-23
+
+- Renamed some configurations
+
+## 0.3.29 - 2018-11-30
+
+- Fix crashes when data schema doesn't contain all of the target Mailchimp list's groups
+
 ## 0.3.28 - 2018-10-29
 
 - Quick fix a typo

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.28"
+version = "0.3.31"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java

@@ -3,12 +3,10 @@ package org.embulk.output.mailchimp;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import org.eclipse.jetty.client.HttpResponseException;
 import org.embulk.base.restclient.jackson.StringJsonParser;
@@ -16,26 +14,27 @@ import org.embulk.config.ConfigException;
 import org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
 import org.embulk.output.mailchimp.helper.MailChimpHelper;
 import org.embulk.output.mailchimp.helper.MailChimpRetryable;
-import org.embulk.output.mailchimp.model.CategoriesResponse;
+import org.embulk.output.mailchimp.model.Category;
 import org.embulk.output.mailchimp.model.ErrorResponse;
-import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
-import org.embulk.output.mailchimp.model.InterestResponse;
-import org.embulk.output.mailchimp.model.InterestsResponse;
+import org.embulk.output.mailchimp.model.Interest;
 import org.embulk.output.mailchimp.model.MergeField;
-import org.embulk.output.mailchimp.model.MergeFields;
 import org.embulk.output.mailchimp.model.ReportResponse;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 
-import java.text.MessageFormat;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.text.MessageFormat.format;
+import static java.util.Arrays.asList;
+
 /**
  * Created by thangnc on 4/25/17.
  */
@@ -68,7 +67,7 @@ public class MailChimpClient
     public ReportResponse push(final ObjectNode node, PluginTask task) throws JsonProcessingException
     {
         try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
-            String response = retryable.post(MessageFormat.format("/lists/{0}", task.getListId()),
+            String response = retryable.post(format("/lists/{0}", task.getListId()),
                                              "application/json;utf-8",
                                              node.toString());
             if (response != null && !response.isEmpty()) {
@@ -90,7 +89,7 @@ public class MailChimpClient
             StringBuilder errorMessage = new StringBuilder();
 
             for (ErrorResponse errorResponse : errorResponses) {
-                errorMessage.append(MessageFormat.format("`{0}` failed cause `{1}`\n",
+                errorMessage.append(format("`{0}` failed cause `{1}`\n",
                                                          MailChimpHelper.maskEmail(errorResponse.getEmailAddress()),
                                                          MailChimpHelper.maskEmail(errorResponse.getError())));
             }
@@ -104,130 +103,128 @@ public class MailChimpClient
      * Extract interest categories by group names. Loop via categories and fetch category details
      * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/interest-categories/#read-get_lists_list_id_interest_categories
      * https://developer.mailchimp.com/documentation/mailchimp/reference/lists/interest-categories/#read-get_lists_list_id_interest_categories_interest_category_id
-     *
-     * @param task the task
-     * @return the map
-     * @throws JsonProcessingException the json processing exception
      */
-    public Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final PluginTask task)
-            throws JsonProcessingException
+    Map<String, Map<String, Interest>> interestsByCategory(final PluginTask task, Schema schema) throws JsonProcessingException
     {
+        if (!task.getGroupingColumns().isPresent() || task.getGroupingColumns().get().isEmpty()) {
+            return Collections.emptyMap();
+        }
         try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
-            Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
-            if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
-                List<String> interestCategoryNames = task.getGroupingColumns().get();
-
-                int count = 100;
-                int offset = 0;
-                int page = 1;
-                boolean hasMore = true;
-                JsonNode response;
-                List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
-
-                while (hasMore) {
-                    String path = MessageFormat.format("/lists/{0}/interest-categories?count={1}&offset={2}",
-                                                       task.getListId(),
-                                                       count,
-                                                       offset);
-                    response = jsonParser.parseJsonObject(retryable.get(path));
-                    InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
-                                                                                               InterestCategoriesResponse.class);
-
-                    allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
-                    if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
-                        offset = count;
-                        page++;
-                    }
-                    else {
-                        hasMore = false;
-                    }
-                }
-
-                Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
-                {
-                    @Override
-                    public String apply(CategoriesResponse input)
-                    {
-                        return input.getTitle().toLowerCase();
-                    }
-                };
-
-                // Transform to a list of available category names and validate with data that user input
-                ImmutableList<String> availableCategories = FluentIterable
-                        .from(allCategoriesResponse)
-                        .transform(function)
-                        .toList();
-
-                for (String category : interestCategoryNames) {
-                    if (!availableCategories.contains(category)) {
-                        throw new ConfigException("Invalid interest category name: '" + category + "'");
-                    }
+            List<Category> categories = fetchCategories(retryable, task.getListId(), task.getGroupingColumns().get());
+            Map<String, Map<String, Interest>> interestsByCategory = new HashMap<>();
+            for (Category category : categories) {
+                // Skip fetching interests if this category isn't specified in the task's grouping column.
+                // Assume task's grouping columns are always in lower case
+                if (!task.getGroupingColumns().get().contains(category.getTitle().toLowerCase())) {
+                    continue;
                 }
+                avoidFloodAPI("Fetching next category's interests", task.getSleepBetweenRequestsMillis());
+                interestsByCategory.put(
+                        category.getTitle().toLowerCase(),
+                        convertInterestCategoryToMap(fetchInterests(retryable, task.getListId(), category.getId())));
+            }
+            return interestsByCategory;
+        }
+    }
 
-                for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
-                    String detailPath = MessageFormat.format("/lists/{0}/interest-categories/{1}/interests",
-                                                             task.getListId(),
-                                                             categoriesResponse.getId());
-                    response = jsonParser.parseJsonObject(retryable.get(detailPath));
-
-                    // Avoid flood MailChimp API
-                    avoidFloodAPI("Fetching next category's interests", task.getSleepBetweenRequestsMillis());
-                    InterestsResponse interestsResponse = mapper.treeToValue(response, InterestsResponse.class);
-                    categories.put(categoriesResponse.getTitle().toLowerCase(),
-                                   convertInterestCategoryToMap(interestsResponse.getInterests()));
+    /**
+     * @throws ConfigException if task having unexist category
+     */
+    private List<Category> fetchCategories(MailChimpRetryable retryable,
+                                           String listId,
+                                           List<String> taskCategories)
+            throws JsonProcessingException
+    {
+        List<Category> categories = fetch(
+                retryable,
+                "/lists/" + listId + "/interest-categories",
+                "categories",
+                Category[].class);
+        // Fail early if one of the task's categories is not exist
+        taskCategories:
+        for (String taskCategoryName : taskCategories) {
+            for (Category category : categories) {
+                // Tasks's configured categories are (implicitly?) case-sensitive
+                if (category.getTitle().toLowerCase().equals(taskCategoryName)) {
+                    continue taskCategories;
                 }
             }
-
-            return categories;
+            throw new ConfigException("Invalid group category name: '" + taskCategoryName + "'");
         }
+        return categories;
     }
 
+    private List<Interest> fetchInterests(MailChimpRetryable retryable,
+                                          String listId,
+                                          String categoryId)
+            throws JsonProcessingException
+    {
+        return fetch(retryable,
+                "/lists/" + listId + "/interest-categories/" + categoryId + "/interests",
+                "interests",
+                Interest[].class);
+    }
     /**
      * Extract merge fields from the list, find correct merge fields from API and put into the map to use
      * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/merge-fields/#read-get_lists_list_id_merge_fields
-     *
-     * @param task the task
-     * @return the map
-     * @throws JsonProcessingException the json processing exception
      */
-    public Map<String, MergeField> extractMergeFieldsFromList(PluginTask task) throws JsonProcessingException
+    Map<String, MergeField> mergeFieldByTag(PluginTask task) throws JsonProcessingException
     {
-        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
-            int count = 100;
-            int offset = 0;
-            int page = 1;
-            boolean hasMore = true;
-            List<MergeField> allMergeFields = new ArrayList<>();
-
-            while (hasMore) {
-                String path = MessageFormat.format("/lists/{0}/merge-fields?count={1}&offset={2}",
-                                                   task.getListId(),
-                                                   count,
-                                                   offset);
-
-                JsonNode response = jsonParser.parseJsonObject(retryable.get(path));
-                MergeFields mergeFields = mapper.treeToValue(response,
-                                                             MergeFields.class);
-
-                allMergeFields.addAll(mergeFields.getMergeFields());
-
-                if (hasMorePage(mergeFields.getTotalItems(), count, page)) {
-                    offset = count;
-                    page++;
-                }
-                else {
-                    hasMore = false;
-                }
-            }
+        return convertMergeFieldToMap(
+                fetch(task,
+                        "/lists/" + task.getListId() + "/merge-fields",
+                        "merge_fields",
+                        MergeField[].class));
+    }
 
-            return convertMergeFieldToMap(allMergeFields);
+    /**
+     * Like {@link MailChimpClient#fetch(MailChimpRetryable, String, String, Class)},
+     * with an automatically created MailchimpRetryable
+     */
+    private <T> List<T> fetch(PluginTask task,
+                              String url,
+                              String recordsAttribute,
+                              Class<T[]> entitiesClass)
+            throws JsonProcessingException
+    {
+        try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
+            return fetch(retryable, url, recordsAttribute, entitiesClass);
+        }
+    }
+    /**
+     * Fetch all (by pagination) records at the target URL,
+     * Assume that endpoint handles `count` and `offset` parameter and have a response scheme of:
+     *     {
+     *        recordsAttribute}: [...],
+     *        "total_items": 10
+     *     }
+     * @param recordsAttribute name of the attribute to extract records inside the response's body.
+     * @param entitiesClass *Array* class of the entity to deserialize into
+     */
+    private <T> List<T> fetch(MailChimpRetryable retryable,
+                              String url,
+                              String recordsAttribute,
+                              Class<T[]> entitiesClass)
+            throws JsonProcessingException
+    {
+        final int batchSize = 100;
+        int offset = 0;
+        List<T> entities = new ArrayList<>();
+        ObjectNode response;
+        do {
+            response = jsonParser.parseJsonObject(
+                    retryable.get(url + "?count=" + batchSize + "&offset=" + offset));
+            entities.addAll(asList(mapper.treeToValue(response.get(recordsAttribute), entitiesClass)));
+            offset += batchSize;
         }
+        while (offset < response.get("total_items").asInt());
+        return entities;
     }
 
     private void findList(final PluginTask task)
     {
         try (MailChimpRetryable retryable = new MailChimpRetryable(task)) {
-            jsonParser.parseJsonObject(retryable.get(MessageFormat.format("/lists/{0}",
+            jsonParser.parseJsonObject(retryable.get(format("/lists/{0}",
                                                                           task.getListId())));
         }
         catch (HttpResponseException hre) {
@@ -235,18 +232,18 @@ public class MailChimpClient
         }
     }
 
-    private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
+    private Map<String, Interest> convertInterestCategoryToMap(final List<Interest> interestList)
     {
-        Function<InterestResponse, String> function = new Function<InterestResponse, String>()
+        Function<Interest, String> function = new Function<Interest, String>()
         {
             @Override
-            public String apply(@Nullable InterestResponse input)
+            public String apply(@Nullable Interest input)
             {
                 return input.getName();
             }
         };
 
-        return Maps.uniqueIndex(FluentIterable.from(interestResponseList)
+        return Maps.uniqueIndex(FluentIterable.from(interestList)
                                         .toList(),
                                 function);
     }

data/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java

@@ -19,9 +19,13 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
+import static com.google.common.base.Joiner.on;
 import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.caseInsensitiveColumnNames;
 import static org.embulk.output.mailchimp.validation.ColumnDataValidator.checkExistColumns;
 
 /**
@@ -39,17 +43,17 @@ public class MailChimpOutputPluginDelegate
     public interface PluginTask
             extends RestClientOutputTaskBase
     {
-        @Config("maximum_retries")
+        @Config("retry_limit")
         @ConfigDefault("6")
-        int getMaximumRetries();
+        int getRetryLimit();
 
-        @Config("initial_retry_interval_millis")
+        @Config("retry_initial_wait_msec")
         @ConfigDefault("1000")
-        int getInitialRetryIntervalMillis();
+        int getRetryInitialWaitMSec();
 
-        @Config("maximum_retry_interval_millis")
+        @Config("max_retry_wait_msec")
         @ConfigDefault("32000")
-        int getMaximumRetryIntervalMillis();
+        int getMaxRetryWaitMSec();
 
         @Config("timeout_millis")
         @ConfigDefault("60000")
@@ -155,6 +159,16 @@ public class MailChimpOutputPluginDelegate
         if (task.getAtomicUpsert()) {
             LOG.info(" Treating upsert as atomic operation");
         }
+
+        // Warn if schema doesn't have the task's grouping column (Group Category)
+        if (task.getGroupingColumns().isPresent()
+                && !task.getGroupingColumns().get().isEmpty()) {
+            Set<String> categoryNames = new HashSet<>(task.getGroupingColumns().get());
+            categoryNames.removeAll(caseInsensitiveColumnNames(schema));
+            if (categoryNames.size() > 0) {
+                LOG.warn("Data schema doesn't contain the task's grouping column(s): {}", on(", ").join(categoryNames));
+            }
+        }
     }
 
     @Override

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -17,7 +17,7 @@ import org.embulk.base.restclient.record.RecordBuffer;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.config.TaskReport;
 import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
-import org.embulk.output.mailchimp.model.InterestResponse;
+import org.embulk.output.mailchimp.model.Interest;
 import org.embulk.output.mailchimp.model.MergeField;
 import org.embulk.output.mailchimp.model.ReportResponse;
 import org.embulk.spi.Column;
@@ -26,8 +26,6 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
-import javax.annotation.Nullable;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -37,9 +35,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
 
-import static com.google.common.base.Joiner.on;
 import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import static java.lang.String.format;
 import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
@@ -65,7 +61,7 @@ public class MailChimpRecordBuffer
     private int errorCount;
     private long totalCount;
     private List<JsonNode> records;
-    private Map<String, Map<String, InterestResponse>> categories;
+    private Map<String, Map<String, Interest>> categories;
     private Map<String, MergeField> availableMergeFields;
     private List<JsonNode> uniqueRecords;
     private List<JsonNode> duplicatedRecords;
@@ -173,19 +169,12 @@ public class MailChimpRecordBuffer
         // Should loop the names and get the id of interest categories.
         // The reason why we put categories validation here because we can not share data between instance.
         if (categories == null) {
-            categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
-
-            Set<String> categoriesNames = categories.keySet();
-            Set<String> columnNames = caseInsensitiveColumnNames();
-            if (!columnNames.containsAll(categoriesNames)) {
-                categoriesNames.removeAll(columnNames);
-                LOG.warn("Data column for category '{}' could not be found", on(", ").join(categoriesNames));
-            }
+            categories = mailChimpClient.interestsByCategory(task, schema);
         }
 
         // Extract merge fields detail
         if (availableMergeFields == null) {
-            availableMergeFields = mailChimpClient.extractMergeFieldsFromList(task);
+            availableMergeFields = mailChimpClient.mergeFieldByTag(task);
         }
 
         // Required merge fields
@@ -288,7 +277,7 @@ public class MailChimpRecordBuffer
                 }
                 List<String> recordInterests = fromCommaSeparatedString(inputValue.get().asText());
                 // `categories` is guaranteed to contain the `category` as it already did an early check
-                Map<String, InterestResponse> availableInterests = categories.get(category);
+                Map<String, Interest> availableInterests = categories.get(category);
 
                 // Only update user-predefined categories if replace interests != true
                 if (!task.getReplaceInterests()) {
@@ -359,21 +348,4 @@ public class MailChimpRecordBuffer
             }
         }
     }
-
-    private Set<String> caseInsensitiveColumnNames()
-    {
-        Set<String> columns = new TreeSet<>(CASE_INSENSITIVE_ORDER);
-        columns.addAll(FluentIterable
-                .from(schema.getColumns())
-                .transform(new Function<Column, String>() {
-                    @Nullable
-                    @Override
-                    public String apply(@Nullable Column col)
-                    {
-                        return col.getName();
-                    }
-                })
-                .toSet());
-        return columns;
-    }
 }

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java

@@ -9,17 +9,23 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
 import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
 
 import javax.annotation.Nullable;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -142,4 +148,20 @@ public final class MailChimpHelper
         }
         return Optional.absent();
     }
+
+    public static Set<String> caseInsensitiveColumnNames(Schema schema)
+    {
+        Set<String> columns = new TreeSet<>(CASE_INSENSITIVE_ORDER);
+        columns.addAll(FluentIterable
+                .from(schema.getColumns())
+                .transform(new Function<Column, String>() {
+                    @Override
+                    public String apply(Column col)
+                    {
+                        return col.getName();
+                    }
+                })
+                .toSet());
+        return columns;
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java

@@ -17,7 +17,6 @@ import org.embulk.spi.Exec;
 import org.embulk.util.retryhelper.jetty92.DefaultJetty92ClientCreator;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
-import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
 import org.slf4j.Logger;
 
 import java.text.MessageFormat;
@@ -44,9 +43,9 @@ public class MailChimpRetryable implements AutoCloseable
 
     public MailChimpRetryable(final PluginTask pluginTask)
     {
-        this.retryHelper = new Jetty92RetryHelper(pluginTask.getMaximumRetries(),
-                                                  pluginTask.getInitialRetryIntervalMillis(),
-                                                  pluginTask.getMaximumRetryIntervalMillis(),
+        this.retryHelper = new Jetty92RetryHelper(pluginTask.getRetryLimit(),
+                                                  pluginTask.getRetryInitialWaitMSec(),
+                                                  pluginTask.getMaxRetryWaitMSec(),
                                                   new DefaultJetty92ClientCreator(pluginTask.getTimeoutMillis(),
                                                                                   pluginTask.getTimeoutMillis()));
         this.pluginTask = pluginTask;
@@ -67,7 +66,7 @@ public class MailChimpRetryable implements AutoCloseable
     {
         try {
             return retryHelper.requestWithRetry(
-                    new StringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
+                    new PatchedStringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
                     new Jetty92SingleRequester()
                     {
                         @Override

data/src/main/java/org/embulk/output/mailchimp/helper/PatchedStringJetty92ResponseEntityReader.java

@@ -0,0 +1,55 @@
+package org.embulk.output.mailchimp.helper;
+
+import com.google.common.io.CharStreams;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.embulk.util.retryhelper.jetty92.Jetty92ResponseReader;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A copy of {@link org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader} with the only
+ * modification is {@link PatchedStringJetty92ResponseEntityReader#getListener()} to return a new instance every time.
+ * This might eventually get fixed upstream (Jetty92RetryHelper aware of Jetty92ResponseReader is stateful),
+ */
+public class PatchedStringJetty92ResponseEntityReader implements Jetty92ResponseReader<String>
+{
+    private InputStreamResponseListener listener;
+    private final long timeoutMillis;
+
+    public PatchedStringJetty92ResponseEntityReader(long timeoutMillis)
+    {
+        this.listener = new InputStreamResponseListener();
+        this.timeoutMillis = timeoutMillis;
+    }
+
+    @Override
+    public final Response.Listener getListener()
+    {
+        this.listener = new InputStreamResponseListener();
+        return this.listener;
+    }
+
+    @Override
+    public final Response getResponse() throws Exception
+    {
+        return this.listener.get(this.timeoutMillis, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public final String readResponseContent() throws Exception
+    {
+        final InputStream inputStream = this.listener.getInputStream();
+        try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream)) {
+            return CharStreams.toString(inputStreamReader);
+        }
+    }
+
+    @Override
+    public final String readResponseContentInString() throws Exception
+    {
+        return this.readResponseContent();
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/{CategoriesResponse.java → Category.java}

@@ -3,7 +3,7 @@ package org.embulk.output.mailchimp.model;
 /**
  * Created by thangnc on 5/5/17.
  */
-public class CategoriesResponse
+public class Category
 {
     private String id;
     private String title;

data/src/main/java/org/embulk/output/mailchimp/model/{InterestResponse.java → Interest.java}

@@ -3,7 +3,7 @@ package org.embulk.output.mailchimp.model;
 /**
  * Created by thangnc on 5/8/17.
  */
-public class InterestResponse
+public class Interest
 {
     private String id;
     private String name;

data/src/test/java/org/embulk/output/mailchimp/TestMailChimpHelper.java

@@ -4,15 +4,20 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Multimap;
 import org.embulk.output.mailchimp.helper.MailChimpHelper;
 import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.caseInsensitiveColumnNames;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.extractMemberStatus;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.maskEmail;
@@ -95,4 +100,12 @@ public class TestMailChimpHelper
         assertEquals("Should be JSON", ObjectNode.class, orderJsonNode(toJsonNode(given), attributes).getClass());
         assertEquals("Should be match", expect, orderJsonNode(toJsonNode(given), attributes).toString());
     }
+
+    @Test
+    public void test_caseInsensitiveColumnNames()
+    {
+        Schema schema = new Schema(ImmutableList.of(
+                new Column(0, "InCONSisTENT", Types.LONG)));
+        assertTrue(caseInsensitiveColumnNames(schema).contains("inConsIstent"));
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.28
+  version: 0.3.31
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-29 00:00:00.000000000 Z
+date: 2019-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,13 +67,12 @@ files:
 - src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java
 - src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java
 - src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java
+- src/main/java/org/embulk/output/mailchimp/helper/PatchedStringJetty92ResponseEntityReader.java
 - src/main/java/org/embulk/output/mailchimp/model/AddressMergeFieldAttribute.java
 - src/main/java/org/embulk/output/mailchimp/model/AuthMethod.java
-- src/main/java/org/embulk/output/mailchimp/model/CategoriesResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/Category.java
 - src/main/java/org/embulk/output/mailchimp/model/ErrorResponse.java
-- src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java
-- src/main/java/org/embulk/output/mailchimp/model/InterestResponse.java
-- src/main/java/org/embulk/output/mailchimp/model/InterestsResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/Interest.java
 - src/main/java/org/embulk/output/mailchimp/model/MemberStatus.java
 - src/main/java/org/embulk/output/mailchimp/model/MergeField.java
 - src/main/java/org/embulk/output/mailchimp/model/MergeFields.java
@@ -89,12 +88,12 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/embulk-output-mailchimp-0.3.31.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
-- classpath/embulk-output-mailchimp-0.3.28.jar
 homepage: https://github.com/treasure-data/embulk-output-mailchimp
 licenses:
 - MIT

data/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java

@@ -1,37 +0,0 @@
-package org.embulk.output.mailchimp.model;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.List;
-
-/**
- * Created by thangnc on 5/8/17.
- */
-public class InterestCategoriesResponse
-{
-    @JsonProperty("categories")
-    private List<CategoriesResponse> categories;
-
-    @JsonProperty("total_items")
-    private int totalItems;
-
-    public List<CategoriesResponse> getCategories()
-    {
-        return categories;
-    }
-
-    public void setCategories(List<CategoriesResponse> categories)
-    {
-        this.categories = categories;
-    }
-
-    public int getTotalItems()
-    {
-        return totalItems;
-    }
-
-    public void setTotalItems(int totalItems)
-    {
-        this.totalItems = totalItems;
-    }
-}

data/src/main/java/org/embulk/output/mailchimp/model/InterestsResponse.java

@@ -1,24 +0,0 @@
-package org.embulk.output.mailchimp.model;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.List;
-
-/**
- * Created by thangnc on 5/8/17.
- */
-public class InterestsResponse
-{
-    @JsonProperty("interests")
-    private List<InterestResponse> interests;
-
-    public List<InterestResponse> getInterests()
-    {
-        return interests;
-    }
-
-    public void setInterests(List<InterestResponse> interests)
-    {
-        this.interests = interests;
-    }
-}