embulk-output-mailchimp 0.2.3 → 0.3.2

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -0,0 +1,174 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.config.ConfigException;
+import org.embulk.output.mailchimp.helper.MailChimpHelper;
+import org.embulk.output.mailchimp.model.CategoriesResponse;
+import org.embulk.output.mailchimp.model.ErrorResponse;
+import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
+import org.embulk.output.mailchimp.model.InterestResponse;
+import org.embulk.output.mailchimp.model.InterestsResponse;
+import org.embulk.output.mailchimp.model.MetaDataResponse;
+import org.embulk.output.mailchimp.model.ReportResponse;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+import java.text.MessageFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.embulk.output.mailchimp.model.AuthMethod.API_KEY;
+import static org.embulk.output.mailchimp.model.AuthMethod.OAUTH;
+
+/**
+ * Created by thangnc on 4/25/17.
+ */
+public class MailChimpRecordBuffer extends MailChimpAbstractRecordBuffer
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpRecordBuffer.class);
+    private MailChimpHttpClient client;
+
+    public MailChimpRecordBuffer(final Schema schema, final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        super(schema, task);
+        client = new MailChimpHttpClient(task);
+    }
+
+    @Override
+    void cleanUp()
+    {
+        client.close();
+    }
+
+    /**
+     * Build an array of email subscribers and batch insert via bulk MailChimp API
+     * Reference: https://developer.mailchimp.com/documentation/mailchimp/reference/lists/#create-post_lists_list_id
+     *
+     * @param node the data
+     * @param task the task
+     * @throws JsonProcessingException the json processing exception
+     */
+    @Override
+    public ReportResponse push(final ObjectNode node, MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException
+    {
+        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}",
+                                               task.getListId());
+
+        JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
+        return getMapper().treeToValue(response, ReportResponse.class);
+    }
+
+    @Override
+    void handleErrors(List<ErrorResponse> errorResponses)
+    {
+        if (!errorResponses.isEmpty()) {
+            StringBuilder errorMessage = new StringBuilder();
+
+            for (ErrorResponse errorResponse : errorResponses) {
+                errorMessage.append(MessageFormat.format("\nEmail `{0}` failed cause `{1}`",
+                                                         MailChimpHelper.maskEmail(errorResponse.getEmailAddress()),
+                                                         MailChimpHelper.maskEmail(errorResponse.getError())));
+            }
+
+            LOG.error(errorMessage.toString());
+        }
+    }
+
+    Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException
+    {
+        Map<String, Map<String, InterestResponse>> categories = new HashMap<>();
+        if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
+            List<String> interestCategoryNames = task.getGroupingColumns().get();
+
+            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
+                                                   task.getListId());
+
+            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
+            InterestCategoriesResponse interestCategoriesResponse = getMapper().treeToValue(response,
+                                                                                            InterestCategoriesResponse.class);
+
+            Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
+            {
+                @Override
+                public String apply(CategoriesResponse input)
+                {
+                    return input.getTitle().toLowerCase();
+                }
+            };
+
+            // Transform to a list of available category names and validate with data that user input
+            ImmutableList<String> availableCategories = FluentIterable
+                    .from(interestCategoriesResponse.getCategories())
+                    .transform(function)
+                    .toList();
+
+            for (String category : interestCategoryNames) {
+                if (!availableCategories.contains(category)) {
+                    throw new ConfigException("Invalid interest category name: '" + category + "'");
+                }
+            }
+
+            for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
+                String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
+                                                             task.getListId(),
+                                                             categoriesResponse.getId());
+                response = client.sendRequest(detailEndpoint, HttpMethod.GET, task);
+                InterestsResponse interestsResponse = getMapper().treeToValue(response, InterestsResponse.class);
+                categories.put(categoriesResponse.getTitle().toLowerCase(),
+                               convertInterestCategoryToMap(interestsResponse.getInterests()));
+            }
+        }
+
+        return categories;
+    }
+
+    @Override
+    String extractDataCenter(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    {
+        if (task.getAuthMethod() == OAUTH) {
+            // Extract data center from meta data URL
+            JsonNode response = client.sendRequest("https://login.mailchimp.com/oauth2/metadata", HttpMethod.GET, task);
+            MetaDataResponse metaDataResponse = getMapper().treeToValue(response, MetaDataResponse.class);
+            return metaDataResponse.getDc();
+        }
+        else if (task.getAuthMethod() == API_KEY && task.getApikey().isPresent()) {
+            // Authenticate and return data center
+            String domain = task.getApikey().get().split("-")[1];
+            String endpoint = MessageFormat.format(mailchimpEndpoint + "/", domain);
+            client.sendRequest(endpoint, HttpMethod.GET, task);
+            return domain;
+        }
+        else {
+            throw new ConfigException("Could not get data center");
+        }
+    }
+
+    private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
+    {
+        Function<InterestResponse, String> function = new Function<InterestResponse, String>()
+        {
+            @Override
+            public String apply(@Nullable InterestResponse input)
+            {
+                return input.getName();
+            }
+        };
+
+        return Maps.uniqueIndex(FluentIterable.from(interestResponseList)
+                                        .toList(),
+                                function);
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java

@@ -0,0 +1,70 @@
+package org.embulk.output.mailchimp.helper;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Function;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * Created by thangnc on 4/26/17.
+ */
+public final class MailChimpHelper
+{
+    private MailChimpHelper()
+    {
+    }
+
+    /**
+     * Mask email string.
+     *
+     * @param email the email
+     * @return the string
+     */
+    public static String maskEmail(final String email)
+    {
+        return email.replaceAll("(?<=.).(?=[^@]*?..@)", "*");
+    }
+
+    /**
+     * This method help to get explicit merge fields with column schema without case-sensitive
+     *
+     * @param s    the s
+     * @param list the list
+     * @return the boolean
+     */
+    public static String containsCaseInsensitive(final String s, final List<String> list)
+    {
+        for (String string : list) {
+            if (string.equalsIgnoreCase(s)) {
+                return string;
+            }
+        }
+
+        return "";
+    }
+
+    /**
+     * Extract member status to validate.
+     *
+     * @param data the data
+     * @return the multimap
+     */
+    public static Multimap<String, JsonNode> extractMemberStatus(final List<JsonNode> data)
+    {
+        Function<JsonNode, String> function = new Function<JsonNode, String>()
+        {
+            @Nullable
+            @Override
+            public String apply(@Nullable JsonNode input)
+            {
+                return input != null ? input.findPath("status").asText() : "";
+            }
+        };
+
+        return Multimaps.index(data, function);
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/AuthMethod.java

@@ -0,0 +1,58 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.embulk.config.ConfigException;
+
+/**
+ * Created by thangnc on 4/17/17.
+ * <p>
+ * MailChimp v3 supports 2 types of auth: OAuth and API key.
+ */
+public enum AuthMethod
+{
+    /**
+     * OAuth2 type
+     */
+    OAUTH("oauth"),
+    /**
+     * API key type
+     */
+    API_KEY("api_key");
+
+    private String type;
+
+    AuthMethod(final String type)
+    {
+        this.type = type;
+    }
+
+    /**
+     * Gets type.
+     *
+     * @return the type
+     */
+    public String getType()
+    {
+        return type;
+    }
+
+    /**
+     * Find by type auth method.
+     *
+     * @param type the type
+     * @return the auth method
+     */
+    @JsonCreator
+    public static AuthMethod findByType(final String type)
+    {
+        for (AuthMethod method : values()) {
+            if (method.getType().equals(type.toLowerCase())) {
+                return method;
+            }
+        }
+
+        throw new ConfigException(
+                String.format("Unknown auth_method '%s'. Supported targets are [api_key, oauth]",
+                              type));
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/CategoriesResponse.java

@@ -0,0 +1,30 @@
+package org.embulk.output.mailchimp.model;
+
+/**
+ * Created by thangnc on 5/5/17.
+ */
+public class CategoriesResponse
+{
+    private String id;
+    private String title;
+
+    public String getId()
+    {
+        return id;
+    }
+
+    public void setId(String id)
+    {
+        this.id = id;
+    }
+
+    public String getTitle()
+    {
+        return title;
+    }
+
+    public void setTitle(String title)
+    {
+        this.title = title;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/ErrorResponse.java

@@ -0,0 +1,56 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * The POJO class of MailChimp v3 error. Each representing an email address that could not be
+ * added to the list or updated and an error message providing more details.
+ * <p>
+ * Created by thangnc on 4/26/17.
+ */
+public class ErrorResponse
+{
+    @JsonProperty("email_address")
+    private String emailAddress;
+    private String error;
+
+    /**
+     * Gets email address.
+     *
+     * @return the email address
+     */
+    public String getEmailAddress()
+    {
+        return emailAddress;
+    }
+
+    /**
+     * Sets email address.
+     *
+     * @param emailAddress the email address
+     */
+    public void setEmailAddress(String emailAddress)
+    {
+        this.emailAddress = emailAddress;
+    }
+
+    /**
+     * Gets error.
+     *
+     * @return the error
+     */
+    public String getError()
+    {
+        return error;
+    }
+
+    /**
+     * Sets error.
+     *
+     * @param error the error
+     */
+    public void setError(String error)
+    {
+        this.error = error;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java

@@ -0,0 +1,24 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * Created by thangnc on 5/8/17.
+ */
+public class InterestCategoriesResponse
+{
+    @JsonProperty("categories")
+    private List<CategoriesResponse> categories;
+
+    public List<CategoriesResponse> getCategories()
+    {
+        return categories;
+    }
+
+    public void setCategories(List<CategoriesResponse> categories)
+    {
+        this.categories = categories;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/InterestResponse.java

@@ -0,0 +1,30 @@
+package org.embulk.output.mailchimp.model;
+
+/**
+ * Created by thangnc on 5/8/17.
+ */
+public class InterestResponse
+{
+    private String id;
+    private String name;
+
+    public String getId()
+    {
+        return id;
+    }
+
+    public void setId(String id)
+    {
+        this.id = id;
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public void setName(String name)
+    {
+        this.name = name;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/InterestsResponse.java

@@ -0,0 +1,24 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * Created by thangnc on 5/8/17.
+ */
+public class InterestsResponse
+{
+    @JsonProperty("interests")
+    private List<InterestResponse> interests;
+
+    public List<InterestResponse> getInterests()
+    {
+        return interests;
+    }
+
+    public void setInterests(List<InterestResponse> interests)
+    {
+        this.interests = interests;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/MemberStatus.java

@@ -0,0 +1,64 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.embulk.config.ConfigException;
+
+/**
+ * Created by thangnc on 4/26/17.
+ */
+public enum MemberStatus
+{
+    /**
+     * Subscribed member status.
+     */
+    SUBSCRIBED("subscribed"),
+    /**
+     * Pending member status. This status will enable to send confirmation email to user
+     */
+    PENDING("pending"),
+    /**
+     * Unsubscribed member status.
+     */
+    UNSUBSCRIBED("unsubscribed"),
+    /**
+     * Cleaned member status. Remove out of list of members but keep log
+     */
+    CLEANED("cleaned");
+
+    private String type;
+
+    MemberStatus(final String type)
+    {
+        this.type = type;
+    }
+
+    /**
+     * Gets type.
+     *
+     * @return the type
+     */
+    public String getType()
+    {
+        return type;
+    }
+
+    /**
+     * Find by type auth method.
+     *
+     * @param type the type
+     * @return the auth method
+     */
+    @JsonCreator
+    public static MemberStatus findByType(final String type)
+    {
+        for (MemberStatus method : values()) {
+            if (method.getType().equals(type.toLowerCase())) {
+                return method;
+            }
+        }
+
+        throw new ConfigException(
+                String.format("Unknown status '%s'. Supported statuses are [subscribed, pending, unsubscribed, cleaned]",
+                              type));
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/MetaDataResponse.java

@@ -0,0 +1,36 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * The POJO class of MailChimp v3 metadata
+ * <p>
+ * Created by thangnc on 5/16/17.
+ */
+public class MetaDataResponse
+{
+    private String dc;
+
+    @JsonProperty("api_endpoint")
+    private String apiEndpoint;
+
+    public String getDc()
+    {
+        return dc;
+    }
+
+    public void setDc(String dc)
+    {
+        this.dc = dc;
+    }
+
+    public String getApiEndpoint()
+    {
+        return apiEndpoint;
+    }
+
+    public void setApiEndpoint(String apiEndpoint)
+    {
+        this.apiEndpoint = apiEndpoint;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/ReportResponse.java

@@ -0,0 +1,105 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * The POJO class of MailChimp v3 response
+ * <p>
+ * Created by thangnc on 4/26/17.
+ */
+public class ReportResponse
+{
+    @JsonProperty("total_created")
+    private int totalCreated;
+
+    @JsonProperty("total_updated")
+    private int totalUpdated;
+
+    @JsonProperty("error_count")
+    private int errorCount;
+
+    @JsonProperty("errors")
+    private List<ErrorResponse> errors;
+
+    /**
+     * The total number of created records matching the query, irrespective of pagination.
+     *
+     * @return the total created
+     */
+    public int getTotalCreated()
+    {
+        return totalCreated;
+    }
+
+    /**
+     * Sets total created.
+     *
+     * @param totalCreated the total created
+     */
+    public void setTotalCreated(int totalCreated)
+    {
+        this.totalCreated = totalCreated;
+    }
+
+    /**
+     * The total number of updated records matching the query, irrespective of pagination.
+     *
+     * @return the total updated
+     */
+    public int getTotalUpdated()
+    {
+        return totalUpdated;
+    }
+
+    /**
+     * Sets total updated.
+     *
+     * @param totalUpdated the total updated
+     */
+    public void setTotalUpdated(int totalUpdated)
+    {
+        this.totalUpdated = totalUpdated;
+    }
+
+    /**
+     * The total number of error records matching the query, irrespective of pagination.
+     *
+     * @return the error count
+     */
+    public int getErrorCount()
+    {
+        return errorCount;
+    }
+
+    /**
+     * Sets error count.
+     *
+     * @param errorCount the error count
+     */
+    public void setErrorCount(int errorCount)
+    {
+        this.errorCount = errorCount;
+    }
+
+    /**
+     * Gets errors.
+     *
+     * @return the errors
+     */
+    public List<ErrorResponse> getErrors()
+    {
+        return errors;
+    }
+
+    /**
+     * Sets errors.
+     *
+     * @param errors the errors
+     */
+    public void setErrors(List<ErrorResponse> errors)
+    {
+        this.errors = errors;
+    }
+}