embulk-output-mailchimp 0.3.13 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7ffeeb484184fcbcb739aef1c871a0c915fca74
-  data.tar.gz: ab5372d39822ac2c231945f1360611ed83f555ec
+  metadata.gz: f89e9a5a69ae081a887e26d19c1a57695d265516
+  data.tar.gz: b3c53d62e62a9127520163a38b053a12b4be50f3
 SHA512:
-  metadata.gz: ff9ea7f57503ac6149b87fb78a67200cfc59fde7fbf66e1bb632d660ab5480a5f4a9e50ddd8d94bdd9d6a4ec2a8b5dc9ee85f4f2c6b1c4f4041e915a991d6483
-  data.tar.gz: 15126516099f6bb739522ec13e9dfcda26ed365a0f686e5e7d9fe7c3dcb37d68cbe97e14a6a760acfa13cfa8c933b20e9f7dad36c5505846bda60817dcb0dde7
+  metadata.gz: d27ebd29fa3ad09b9c942ef2b2276db918f7ab38c2ff8fb819ed4cbfe7ecbea3e4bd25c1926cef43ed317c2d431c030916f015f8c8aacd24f6a125360a8f0326
+  data.tar.gz: 376d7dadd0bec805f68d891930d0c1ebb60d618bbeb55519e90b80e92535826aa7e278b7c40d3d115c22a225aa56aed0ef46fff65ad1a01dae5d7022b57e1f6a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.14 - 2017-06-26
+- Make clear the log message and renamed class `MailChimpClient` [#29](https://github.com/treasure-data/embulk-output-mailchimp/pull/29)
+
 ## 0.3.13 - 2017-06-16
 - Upgraded `embulk-base-restclient` to v0.5.3 and fixed hang job [#28](https://github.com/treasure-data/embulk-output-mailchimp/pull/28)
 

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.13"
+version = "0.3.14"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java

@@ -0,0 +1,235 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.config.ConfigException;
+import org.embulk.output.mailchimp.helper.MailChimpHelper;
+import org.embulk.output.mailchimp.model.CategoriesResponse;
+import org.embulk.output.mailchimp.model.ErrorResponse;
+import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
+import org.embulk.output.mailchimp.model.InterestResponse;
+import org.embulk.output.mailchimp.model.InterestsResponse;
+import org.embulk.output.mailchimp.model.MergeField;
+import org.embulk.output.mailchimp.model.MergeFields;
+import org.embulk.output.mailchimp.model.MetaDataResponse;
+import org.embulk.output.mailchimp.model.ReportResponse;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.embulk.output.mailchimp.model.AuthMethod.API_KEY;
+import static org.embulk.output.mailchimp.model.AuthMethod.OAUTH;
+
+/**
+ * Created by thangnc on 4/25/17.
+ */
+public class MailChimpClient
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpClient.class);
+    private static final String API_VERSION = "3.0";
+    private static String mailchimpEndpoint;
+    private MailChimpHttpClient client;
+    private final ObjectMapper mapper;
+
+    /**
+     * Instantiates a new Mail chimp client.
+     *
+     * @param task the task
+     */
+    public MailChimpClient(final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        mailchimpEndpoint = Joiner.on("/").join("https://{0}.api.mailchimp.com", API_VERSION);
+        this.client = new MailChimpHttpClient(task);
+        this.mapper = new ObjectMapper()
+                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+        extractDataCenter(task);
+    }
+
+    /**
+     * Build an array of email subscribers and batch insert via bulk MailChimp API
+     * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/#create-post_lists_list_id
+     *
+     * @param node the data
+     * @param task the task
+     * @return the report response
+     * @throws JsonProcessingException the json processing exception
+     */
+    ReportResponse push(final ObjectNode node, MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException
+    {
+        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
+                                               task.getListId());
+
+        JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
+        return mapper.treeToValue(response, ReportResponse.class);
+    }
+
+    /**
+     * Handle detail errors after call bulk MailChimp API
+     *
+     * @param errorResponses the error responses
+     */
+    void handleErrors(List<ErrorResponse> errorResponses)
+    {
+        if (!errorResponses.isEmpty()) {
+            StringBuilder errorMessage = new StringBuilder();
+
+            for (ErrorResponse errorResponse : errorResponses) {
+                errorMessage.append(MessageFormat.format("`{0}` failed cause `{1}`\n",
+                                                         MailChimpHelper.maskEmail(errorResponse.getEmailAddress()),
+                                                         MailChimpHelper.maskEmail(errorResponse.getError())));
+            }
+
+            LOG.error("Error response from MailChimp: ");
+            LOG.error(errorMessage.toString());
+        }
+    }
+
+    /**
+     * Extract interest categories by group names. Loop via categories and fetch category details
+     * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/interest-categories/#read-get_lists_list_id_interest_categories
+     * https://developer.mailchimp.com/documentation/mailchimp/reference/lists/interest-categories/#read-get_lists_list_id_interest_categories_interest_category_id
+     *
+     * @param task the task
+     * @return the map
+     * @throws JsonProcessingException the json processing exception
+     */
+    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException
+    {
+        Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
+        if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
+            List<String> interestCategoryNames = task.getGroupingColumns().get();
+
+            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
+                                                   task.getListId());
+
+            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
+            InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
+                                                                                       InterestCategoriesResponse.class);
+
+            Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
+            {
+                @Override
+                public String apply(CategoriesResponse input)
+                {
+                    return input.getTitle().toLowerCase();
+                }
+            };
+
+            // Transform to a list of available category names and validate with data that user input
+            ImmutableList<String> availableCategories = FluentIterable
+                    .from(interestCategoriesResponse.getCategories())
+                    .transform(function)
+                    .toList();
+
+            for (String category : interestCategoryNames) {
+                if (!availableCategories.contains(category)) {
+                    throw new ConfigException("Invalid interest category name: '" + category + "'");
+                }
+            }
+
+            for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
+                String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
+                                                             task.getListId(),
+                                                             categoriesResponse.getId());
+                response = client.sendRequest(detailEndpoint, HttpMethod.GET, task);
+                InterestsResponse interestsResponse = mapper.treeToValue(response, InterestsResponse.class);
+                categories.put(categoriesResponse.getTitle().toLowerCase(),
+                               convertInterestCategoryToMap(interestsResponse.getInterests()));
+            }
+        }
+
+        return categories;
+    }
+
+    /**
+     * Extract merge fields from the list, find correct merge fields from API and put into the map to use
+     * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/merge-fields/#read-get_lists_list_id_merge_fields
+     *
+     * @param task the task
+     * @return the map
+     * @throws JsonProcessingException the json processing exception
+     */
+    Map<String, MergeField> extractMergeFieldsFromList(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    {
+        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields",
+                                               task.getListId());
+        JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
+        MergeFields mergeFields = mapper.treeToValue(response,
+                                                     MergeFields.class);
+        return convertMergeFieldToMap(mergeFields.getMergeFields());
+    }
+
+    private void extractDataCenter(MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        try {
+            if (task.getAuthMethod() == OAUTH) {
+                // Extract data center from meta data URL
+                JsonNode response = client.sendRequest("https://login.mailchimp.com/oauth2/metadata", HttpMethod.GET, task);
+                MetaDataResponse metaDataResponse = mapper.treeToValue(response, MetaDataResponse.class);
+                mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, metaDataResponse.getDc());
+            }
+            else if (task.getAuthMethod() == API_KEY && task.getApikey().isPresent()) {
+                // Authenticate and return data center
+                String domain = task.getApikey().get().split("-")[1];
+                String endpoint = MessageFormat.format(mailchimpEndpoint + "/", domain);
+                client.sendRequest(endpoint, HttpMethod.GET, task);
+                mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, domain);
+            }
+        }
+        catch (Exception e) {
+            throw new ConfigException("Could not get data center", e);
+        }
+    }
+
+    private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
+    {
+        Function<InterestResponse, String> function = new Function<InterestResponse, String>()
+        {
+            @Override
+            public String apply(@Nullable InterestResponse input)
+            {
+                return input.getName();
+            }
+        };
+
+        return Maps.uniqueIndex(FluentIterable.from(interestResponseList)
+                                        .toList(),
+                                function);
+    }
+
+    private Map<String, MergeField> convertMergeFieldToMap(final List<MergeField> mergeFieldList)
+    {
+        Function<MergeField, String> function = new Function<MergeField, String>()
+        {
+            @Nullable
+            @Override
+            public String apply(@Nullable MergeField input)
+            {
+                return input.getTag().toLowerCase();
+            }
+        };
+
+        return Maps.uniqueIndex(FluentIterable.from(mergeFieldList)
+                                        .toList(),
+                                function);
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -1,208 +1,276 @@
 package org.embulk.output.mailchimp;
 
+import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
+import com.google.common.base.Throwables;
 import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import org.eclipse.jetty.http.HttpMethod;
-import org.embulk.config.ConfigException;
-import org.embulk.output.mailchimp.helper.MailChimpHelper;
-import org.embulk.output.mailchimp.model.CategoriesResponse;
-import org.embulk.output.mailchimp.model.ErrorResponse;
-import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
+import org.embulk.base.restclient.jackson.JacksonServiceRecord;
+import org.embulk.base.restclient.record.RecordBuffer;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.config.TaskReport;
+import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
 import org.embulk.output.mailchimp.model.InterestResponse;
-import org.embulk.output.mailchimp.model.InterestsResponse;
 import org.embulk.output.mailchimp.model.MergeField;
-import org.embulk.output.mailchimp.model.MergeFields;
-import org.embulk.output.mailchimp.model.MetaDataResponse;
 import org.embulk.output.mailchimp.model.ReportResponse;
+import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
-import javax.annotation.Nullable;
-
-import java.text.MessageFormat;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.embulk.output.mailchimp.model.AuthMethod.API_KEY;
-import static org.embulk.output.mailchimp.model.AuthMethod.OAUTH;
+import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.fromCommaSeparatedString;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.orderJsonNode;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
+import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
+import static org.embulk.output.mailchimp.model.MemberStatus.SUBSCRIBED;
 
 /**
- * Created by thangnc on 4/25/17.
+ * Created by thangnc on 4/14/17.
  */
-public class MailChimpRecordBuffer extends MailChimpAbstractRecordBuffer
+public class MailChimpRecordBuffer
+        extends RecordBuffer
 {
     private static final Logger LOG = Exec.getLogger(MailChimpRecordBuffer.class);
-    private MailChimpHttpClient client;
-
-    public MailChimpRecordBuffer(final Schema schema, final MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        super(schema, task);
-        client = new MailChimpHttpClient(task);
-    }
-
-    @Override
-    public void finish()
-    {
-    }
-
-    @Override
-    public void close()
-    {
-    }
+    private static final int MAX_RECORD_PER_BATCH_REQUEST = 500;
+    private final MailChimpOutputPluginDelegate.PluginTask task;
+    private final MailChimpClient mailChimpClient;
+    private final ObjectMapper mapper;
+    private final Schema schema;
+    private int requestCount;
+    private long totalCount;
+    private List<JsonNode> records;
+    private Map<String, Map<String, InterestResponse>> categories;
+    private Map<String, MergeField> availableMergeFields;
 
     /**
-     * Build an array of email subscribers and batch insert via bulk MailChimp API
-     * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/#create-post_lists_list_id
+     * Instantiates a new Mail chimp abstract record buffer.
      *
-     * @param node the data
-     * @param task the task
-     * @throws JsonProcessingException the json processing exception
+     * @param schema the schema
+     * @param task   the task
      */
-    @Override
-    public ReportResponse push(final ObjectNode node, MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException
+    public MailChimpRecordBuffer(final Schema schema, final PluginTask task)
     {
-        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
-                                               task.getListId());
-
-        JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
-        return getMapper().treeToValue(response, ReportResponse.class);
+        this.schema = schema;
+        this.task = task;
+        this.mapper = new ObjectMapper()
+                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+        this.records = new ArrayList<>();
+        this.categories = new HashMap<>();
+        this.mailChimpClient = new MailChimpClient(task);
     }
 
     @Override
-    void handleErrors(List<ErrorResponse> errorResponses)
+    public void bufferRecord(ServiceRecord serviceRecord)
     {
-        if (!errorResponses.isEmpty()) {
-            StringBuilder errorMessage = new StringBuilder();
+        JacksonServiceRecord jacksonServiceRecord;
 
-            for (ErrorResponse errorResponse : errorResponses) {
-                errorMessage.append(MessageFormat.format("\nEmail `{0}` failed cause `{1}`",
-                                                         MailChimpHelper.maskEmail(errorResponse.getEmailAddress()),
-                                                         MailChimpHelper.maskEmail(errorResponse.getError())));
-            }
+        try {
+            jacksonServiceRecord = (JacksonServiceRecord) serviceRecord;
+            JsonNode record = mapper.readTree(jacksonServiceRecord.toString()).get("record");
 
-            LOG.error(errorMessage.toString());
-        }
-    }
+            requestCount++;
+            totalCount++;
 
-    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException
-    {
-        Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
-        if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
-            List<String> interestCategoryNames = task.getGroupingColumns().get();
+            records.add(record);
+            if (requestCount >= MAX_RECORD_PER_BATCH_REQUEST) {
+                ObjectNode subcribers = processSubcribers(records, task);
+                ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
 
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
-                                                   task.getListId());
-
-            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-            InterestCategoriesResponse interestCategoriesResponse = getMapper().treeToValue(response,
-                                                                                            InterestCategoriesResponse.class);
-
-            Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
-            {
-                @Override
-                public String apply(CategoriesResponse input)
-                {
-                    return input.getTitle().toLowerCase();
+                if (totalCount % 1000 == 0) {
+                    LOG.info("Pushed {} records", totalCount);
                 }
-            };
-
-            // Transform to a list of available category names and validate with data that user input
-            ImmutableList<String> availableCategories = FluentIterable
-                    .from(interestCategoriesResponse.getCategories())
-                    .transform(function)
-                    .toList();
 
-            for (String category : interestCategoryNames) {
-                if (!availableCategories.contains(category)) {
-                    throw new ConfigException("Invalid interest category name: '" + category + "'");
-                }
-            }
+                LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
+                         reportResponse.getTotalCreated(),
+                         reportResponse.getTotalUpdated(),
+                         reportResponse.getErrorCount());
+                mailChimpClient.handleErrors(reportResponse.getErrors());
 
-            for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
-                String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
-                                                             task.getListId(),
-                                                             categoriesResponse.getId());
-                response = client.sendRequest(detailEndpoint, HttpMethod.GET, task);
-                InterestsResponse interestsResponse = getMapper().treeToValue(response, InterestsResponse.class);
-                categories.put(categoriesResponse.getTitle().toLowerCase(),
-                               convertInterestCategoryToMap(interestsResponse.getInterests()));
+                records = new ArrayList<>();
+                requestCount = 0;
             }
         }
-
-        return categories;
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
+        }
+        catch (ClassCastException ex) {
+            throw new RuntimeException(ex);
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
     }
 
     @Override
-    String extractDataCenter(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    public TaskReport commitWithTaskReportUpdated(TaskReport taskReport)
     {
-        if (task.getAuthMethod() == OAUTH) {
-            // Extract data center from meta data URL
-            JsonNode response = client.sendRequest("https://login.mailchimp.com/oauth2/metadata", HttpMethod.GET, task);
-            MetaDataResponse metaDataResponse = getMapper().treeToValue(response, MetaDataResponse.class);
-            return metaDataResponse.getDc();
+        try {
+            if (records.size() > 0) {
+                ObjectNode subcribers = processSubcribers(records, task);
+                ReportResponse reportResponse = mailChimpClient.push(subcribers, task);
+                LOG.info("Pushed {} records", records.size());
+                LOG.info("Response from MailChimp: {} records created, {} records updated, {} records failed",
+                         reportResponse.getTotalCreated(),
+                         reportResponse.getTotalUpdated(),
+                         reportResponse.getErrorCount());
+                mailChimpClient.handleErrors(reportResponse.getErrors());
+            }
+
+            return Exec.newTaskReport().set("pushed", totalCount);
         }
-        else if (task.getAuthMethod() == API_KEY && task.getApikey().isPresent()) {
-            // Authenticate and return data center
-            String domain = task.getApikey().get().split("-")[1];
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/", domain);
-            client.sendRequest(endpoint, HttpMethod.GET, task);
-            return domain;
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
         }
-        else {
-            throw new ConfigException("Could not get data center");
+        catch (Exception ex) {
+            throw Throwables.propagate(ex);
         }
     }
 
     @Override
-    Map<String, MergeField> extractMergeFieldsFromList(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    public void finish()
+    {
+        // Do not close here
+    }
+
+    @Override
+    public void close()
     {
-        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields",
-                                               task.getListId());
-        JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-        MergeFields mergeFields = getMapper().treeToValue(response,
-                                                          MergeFields.class);
-        return convertMergeFieldToMap(mergeFields.getMergeFields());
+        // Do not implement
     }
 
-    private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
+    /**
+     * Receive data and build payload json that contains subscribers
+     *
+     * @param data the data
+     * @param task the task
+     * @return the object node
+     */
+    private ObjectNode processSubcribers(final List<JsonNode> data, final PluginTask task)
+            throws JsonProcessingException
     {
-        Function<InterestResponse, String> function = new Function<InterestResponse, String>()
-        {
-            @Override
-            public String apply(@Nullable InterestResponse input)
-            {
-                return input.getName();
-            }
-        };
+        LOG.info("Start to process subscriber data");
+
+        // Should loop the names and get the id of interest categories.
+        // The reason why we put categories validation here because we can not share data between instance.
+        categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
+
+        // Extract merge fields detail
+        availableMergeFields = mailChimpClient.extractMergeFieldsFromList(task);
+
+        // Required merge fields
+        Map<String, String> map = new HashMap<>();
+        map.put("FNAME", task.getFnameColumn());
+        map.put("LNAME", task.getLnameColumn());
 
-        return Maps.uniqueIndex(FluentIterable.from(interestResponseList)
-                                        .toList(),
-                                function);
+        List<JsonNode> subscribersList = FluentIterable.from(data)
+                .transform(contactMapper(map))
+                .toList();
+
+        ObjectNode subscribers = JsonNodeFactory.instance.objectNode();
+        subscribers.putArray("members").addAll(subscribersList);
+        subscribers.put("update_existing", task.getUpdateExisting());
+        return subscribers;
     }
 
-    private Map<String, MergeField> convertMergeFieldToMap(final List<MergeField> mergeFieldList)
+    private Function<JsonNode, JsonNode> contactMapper(final Map<String, String> allowColumns)
     {
-        Function<MergeField, String> function = new Function<MergeField, String>()
+        return new Function<JsonNode, JsonNode>()
         {
-            @Nullable
             @Override
-            public String apply(@Nullable MergeField input)
+            public JsonNode apply(JsonNode input)
             {
-                return input.getTag().toLowerCase();
+                ObjectNode property = JsonNodeFactory.instance.objectNode();
+                property.put("email_address", input.findPath(task.getEmailColumn()).asText());
+                property.put("status", task.getDoubleOptIn() ? PENDING.getType() : SUBSCRIBED.getType());
+                ObjectNode mergeFields = JsonNodeFactory.instance.objectNode();
+                for (String allowColumn : allowColumns.keySet()) {
+                    String value = input.hasNonNull(allowColumns.get(allowColumn)) ? input.findValue(allowColumns.get(allowColumn)).asText() : "";
+                    mergeFields.put(allowColumn, value);
+                }
+
+                // Update additional merge fields if exist
+                if (task.getMergeFields().isPresent() && !task.getMergeFields().get().isEmpty()) {
+                    for (final Column column : schema.getColumns()) {
+                        if (!"".equals(containsCaseInsensitive(column.getName(), task.getMergeFields().get()))) {
+                            String value = input.hasNonNull(column.getName()) ? input.findValue(column.getName()).asText() : "";
+
+                            // Try to convert to Json from string with the merge field's type is address
+                            if (availableMergeFields.get(column.getName()).getType()
+                                    .equals(MergeField.MergeFieldType.ADDRESS.getType())) {
+                                JsonNode addressNode = toJsonNode(value);
+                                if (addressNode instanceof NullNode) {
+                                    mergeFields.put(column.getName().toUpperCase(), value);
+                                }
+                                else {
+                                    mergeFields.set(column.getName().toUpperCase(),
+                                                    orderJsonNode(addressNode, AddressMergeFieldAttribute.values()));
+                                }
+                            }
+                            else {
+                                mergeFields.put(column.getName().toUpperCase(), value);
+                            }
+                        }
+                    }
+                }
+
+                property.set("merge_fields", mergeFields);
+
+                // Update interest categories if exist
+                if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
+                    property.set("interests", buildInterestCategories(task, input));
+                }
+
+                return property;
             }
         };
+    }
+
+    private ObjectNode buildInterestCategories(final MailChimpOutputPluginDelegate.PluginTask task,
+                                               final JsonNode input)
+    {
+        ObjectNode interests = JsonNodeFactory.instance.objectNode();
+
+        for (String category : task.getGroupingColumns().get()) {
+            String inputValue = input.findValue(category).asText();
+            List<String> interestValues = fromCommaSeparatedString(inputValue);
+            Map<String, InterestResponse> availableCategories = categories.get(category);
+
+            // Only update user-predefined categories if replace interests != true
+            if (!task.getReplaceInterests()) {
+                for (String interestValue : interestValues) {
+                    if (availableCategories.get(interestValue) != null) {
+                        interests.put(availableCategories.get(interestValue).getId(), true);
+                    }
+                }
+            } // Otherwise, force update all categories include user-predefined categories
+            else if (task.getReplaceInterests()) {
+                for (String availableCategory : availableCategories.keySet()) {
+                    if (interestValues.contains(availableCategory)) {
+                        interests.put(availableCategories.get(availableCategory).getId(), true);
+                    }
+                    else {
+                        interests.put(availableCategories.get(availableCategory).getId(), false);
+                    }
+                }
+            }
+        }
 
-        return Maps.uniqueIndex(FluentIterable.from(mergeFieldList)
-                                        .toList(),
-                                function);
+        return interests;
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.14
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-16 00:00:00.000000000 Z
+date: 2017-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,7 +61,7 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/mailchimp.rb
-- src/main/java/org/embulk/output/mailchimp/MailChimpAbstractRecordBuffer.java
+- src/main/java/org/embulk/output/mailchimp/MailChimpClient.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpOutputPlugin.java
 - src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java
@@ -89,7 +89,7 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-output-mailchimp-0.3.13.jar
+- classpath/embulk-output-mailchimp-0.3.14.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar

data/src/main/java/org/embulk/output/mailchimp/MailChimpAbstractRecordBuffer.java

@@ -1,338 +0,0 @@
-package org.embulk.output.mailchimp;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.NullNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Function;
-import com.google.common.base.Throwables;
-import com.google.common.collect.FluentIterable;
-import org.embulk.base.restclient.jackson.JacksonServiceRecord;
-import org.embulk.base.restclient.record.RecordBuffer;
-import org.embulk.base.restclient.record.ServiceRecord;
-import org.embulk.config.TaskReport;
-import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
-import org.embulk.output.mailchimp.model.ErrorResponse;
-import org.embulk.output.mailchimp.model.InterestResponse;
-import org.embulk.output.mailchimp.model.MergeField;
-import org.embulk.output.mailchimp.model.ReportResponse;
-import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
-import org.embulk.spi.Schema;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
-import static org.embulk.output.mailchimp.helper.MailChimpHelper.fromCommaSeparatedString;
-import static org.embulk.output.mailchimp.helper.MailChimpHelper.orderJsonNode;
-import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
-import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
-import static org.embulk.output.mailchimp.model.MemberStatus.SUBSCRIBED;
-
-/**
- * Created by thangnc on 4/14/17.
- */
-public abstract class MailChimpAbstractRecordBuffer
-        extends RecordBuffer
-{
-    private static final Logger LOG = Exec.getLogger(MailChimpAbstractRecordBuffer.class);
-    private static final int MAX_RECORD_PER_BATCH_REQUEST = 500;
-    /**
-     * The constant mailchimpEndpoint.
-     */
-    protected static String mailchimpEndpoint = "https://{0}.api.mailchimp.com/3.0";
-    private final MailChimpOutputPluginDelegate.PluginTask task;
-    private final ObjectMapper mapper;
-    private final Schema schema;
-    private int requestCount;
-    private long totalCount;
-    private List<JsonNode> records;
-    private Map<String, Map<String, InterestResponse>> categories;
-    private Map<String, MergeField> availableMergeFields;
-
-    /**
-     * Instantiates a new Mail chimp abstract record buffer.
-     *
-     * @param schema the schema
-     * @param task   the task
-     */
-    public MailChimpAbstractRecordBuffer(final Schema schema, final MailChimpOutputPluginDelegate.PluginTask task)
-    {
-        this.schema = schema;
-        this.task = task;
-        this.mapper = new ObjectMapper()
-                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
-        this.records = new ArrayList<>();
-        this.categories = new HashMap<>();
-    }
-
-    @Override
-    public void bufferRecord(ServiceRecord serviceRecord)
-    {
-        JacksonServiceRecord jacksonServiceRecord;
-
-        try {
-            jacksonServiceRecord = (JacksonServiceRecord) serviceRecord;
-            JsonNode record = mapper.readTree(jacksonServiceRecord.toString()).get("record");
-
-            requestCount++;
-            totalCount++;
-
-            records.add(record);
-            if (requestCount >= MAX_RECORD_PER_BATCH_REQUEST) {
-                ObjectNode subcribers = processSubcribers(records, task);
-                ReportResponse reportResponse = push(subcribers, task);
-
-                if (totalCount % 1000 == 0) {
-                    LOG.info("Pushed {} records", totalCount);
-                }
-
-                LOG.info("{} records created, {} records updated, {} records failed",
-                         reportResponse.getTotalCreated(),
-                         reportResponse.getTotalUpdated(),
-                         reportResponse.getErrorCount());
-                handleErrors(reportResponse.getErrors());
-
-                records = new ArrayList<>();
-                requestCount = 0;
-            }
-        }
-        catch (JsonProcessingException jpe) {
-            throw new DataException(jpe);
-        }
-        catch (ClassCastException ex) {
-            throw new RuntimeException(ex);
-        }
-        catch (IOException ex) {
-            throw Throwables.propagate(ex);
-        }
-    }
-
-    @Override
-    public TaskReport commitWithTaskReportUpdated(TaskReport taskReport)
-    {
-        try {
-            if (records.size() > 0) {
-                ObjectNode subcribers = processSubcribers(records, task);
-                ReportResponse reportResponse = push(subcribers, task);
-                LOG.info("Pushed {} records", records.size());
-                LOG.info("{} records created, {} records updated, {} records failed",
-                         reportResponse.getTotalCreated(),
-                         reportResponse.getTotalUpdated(),
-                         reportResponse.getErrorCount());
-                handleErrors(reportResponse.getErrors());
-            }
-
-            return Exec.newTaskReport().set("pushed", totalCount);
-        }
-        catch (JsonProcessingException jpe) {
-            throw new DataException(jpe);
-        }
-        catch (Exception ex) {
-            throw Throwables.propagate(ex);
-        }
-    }
-
-    /**
-     * Receive data and build payload json that contains subscribers
-     *
-     * @param data the data
-     * @param task the task
-     * @return the object node
-     */
-    ObjectNode processSubcribers(final List<JsonNode> data, final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException
-    {
-        LOG.info("Start to process subscriber data");
-        // Extract data center from meta data URL
-        String dc = extractDataCenter(task);
-        mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, dc);
-
-        // Should loop the names and get the id of interest categories.
-        // The reason why we put categories validation here because we can not share data between instance.
-        categories = extractInterestCategoriesByGroupNames(task);
-
-        // Extract merge fields detail
-        availableMergeFields = extractMergeFieldsFromList(task);
-
-        // Required merge fields
-        Map<String, String> map = new HashMap<>();
-        map.put("FNAME", task.getFnameColumn());
-        map.put("LNAME", task.getLnameColumn());
-
-        List<JsonNode> subscribersList = FluentIterable.from(data)
-                .transform(contactMapper(map))
-                .toList();
-
-        ObjectNode subscribers = JsonNodeFactory.instance.objectNode();
-        subscribers.putArray("members").addAll(subscribersList);
-        subscribers.put("update_existing", task.getUpdateExisting());
-        return subscribers;
-    }
-
-    /**
-     * Gets mapper.
-     *
-     * @return the mapper
-     */
-    public ObjectMapper getMapper()
-    {
-        return mapper;
-    }
-
-    /**
-     * Gets categories.
-     *
-     * @return the categories
-     */
-    public Map<String, Map<String, InterestResponse>> getCategories()
-    {
-        return categories;
-    }
-
-    /**
-     * Push payload data to MailChimp API and get @{@link ReportResponse}
-     *
-     * @param node the content
-     * @param task the task
-     * @return the report response
-     * @throws JsonProcessingException the json processing exception
-     */
-    abstract ReportResponse push(final ObjectNode node, final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException;
-
-    /**
-     * Handle @{@link ErrorResponse} from MailChimp API if exists.
-     *
-     * @param errorResponses the error responses
-     */
-    abstract void handleErrors(final List<ErrorResponse> errorResponses);
-
-    /**
-     * Find interest category ids by pre-defined group name which user input.
-     *
-     * @param task the task
-     * @return the map
-     * @throws JsonProcessingException the json processing exception
-     */
-    abstract Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException;
-
-    /**
-     * Extract data center from MailChimp v3 metadata.
-     *
-     * @param task the task
-     * @return the string
-     * @throws JsonProcessingException the json processing exception
-     */
-    abstract String extractDataCenter(final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException;
-
-    /**
-     * Extract all merge fields from MailChimp list.
-     *
-     * @param task the task
-     * @return the map
-     * @throws JsonProcessingException the json processing exception
-     */
-    abstract Map<String, MergeField> extractMergeFieldsFromList(final MailChimpOutputPluginDelegate.PluginTask task)
-            throws JsonProcessingException;
-
-    private Function<JsonNode, JsonNode> contactMapper(final Map<String, String> allowColumns)
-    {
-        return new Function<JsonNode, JsonNode>()
-        {
-            @Override
-            public JsonNode apply(JsonNode input)
-            {
-                ObjectNode property = JsonNodeFactory.instance.objectNode();
-                property.put("email_address", input.findPath(task.getEmailColumn()).asText());
-                property.put("status", task.getDoubleOptIn() ? PENDING.getType() : SUBSCRIBED.getType());
-                ObjectNode mergeFields = JsonNodeFactory.instance.objectNode();
-                for (String allowColumn : allowColumns.keySet()) {
-                    String value = input.hasNonNull(allowColumns.get(allowColumn)) ? input.findValue(allowColumns.get(allowColumn)).asText() : "";
-                    mergeFields.put(allowColumn, value);
-                }
-
-                // Update additional merge fields if exist
-                if (task.getMergeFields().isPresent() && !task.getMergeFields().get().isEmpty()) {
-                    for (final Column column : schema.getColumns()) {
-                        if (!"".equals(containsCaseInsensitive(column.getName(), task.getMergeFields().get()))) {
-                            String value = input.hasNonNull(column.getName()) ? input.findValue(column.getName()).asText() : "";
-
-                            // Try to convert to Json from string with the merge field's type is address
-                            if (availableMergeFields.get(column.getName()).getType()
-                                    .equals(MergeField.MergeFieldType.ADDRESS.getType())) {
-                                JsonNode addressNode = toJsonNode(value);
-                                if (addressNode instanceof NullNode) {
-                                    mergeFields.put(column.getName().toUpperCase(), value);
-                                }
-                                else {
-                                    mergeFields.set(column.getName().toUpperCase(),
-                                                    orderJsonNode(addressNode, AddressMergeFieldAttribute.values()));
-                                }
-                            }
-                            else {
-                                mergeFields.put(column.getName().toUpperCase(), value);
-                            }
-                        }
-                    }
-                }
-
-                property.set("merge_fields", mergeFields);
-
-                // Update interest categories if exist
-                if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
-                    property.set("interests", buildInterestCategories(task, input));
-                }
-
-                return property;
-            }
-        };
-    }
-
-    private ObjectNode buildInterestCategories(final MailChimpOutputPluginDelegate.PluginTask task,
-                                               final JsonNode input)
-    {
-        ObjectNode interests = JsonNodeFactory.instance.objectNode();
-
-        for (String category : task.getGroupingColumns().get()) {
-            String inputValue = input.findValue(category).asText();
-            List<String> interestValues = fromCommaSeparatedString(inputValue);
-            Map<String, InterestResponse> availableCategories = categories.get(category);
-
-            // Only update user-predefined categories if replace interests != true
-            if (!task.getReplaceInterests()) {
-                for (String interestValue : interestValues) {
-                    if (availableCategories.get(interestValue) != null) {
-                        interests.put(availableCategories.get(interestValue).getId(), true);
-                    }
-                }
-            } // Otherwise, force update all categories include user-predefined categories
-            else if (task.getReplaceInterests()) {
-                for (String availableCategory : availableCategories.keySet()) {
-                    if (interestValues.contains(availableCategory)) {
-                        interests.put(availableCategories.get(availableCategory).getId(), true);
-                    }
-                    else {
-                        interests.put(availableCategories.get(availableCategory).getId(), false);
-                    }
-                }
-            }
-        }
-
-        return interests;
-    }
-}