embulk-output-mailchimp 0.2.3 → 0.3.2

data/src/main/java/org/embulk/output/mailchimp/MailChimpAbstractRecordBuffer.java

@@ -0,0 +1,320 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.FluentIterable;
+import org.embulk.base.restclient.jackson.JacksonServiceRecord;
+import org.embulk.base.restclient.record.RecordBuffer;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.config.TaskReport;
+import org.embulk.output.mailchimp.model.ErrorResponse;
+import org.embulk.output.mailchimp.model.InterestResponse;
+import org.embulk.output.mailchimp.model.ReportResponse;
+import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
+import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
+import static org.embulk.output.mailchimp.model.MemberStatus.SUBSCRIBED;
+
+/**
+ * Created by thangnc on 4/14/17.
+ */
+public abstract class MailChimpAbstractRecordBuffer
+        extends RecordBuffer
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpAbstractRecordBuffer.class);
+    private static final int MAX_RECORD_PER_BATCH_REQUEST = 500;
+    /**
+     * The constant mailchimpEndpoint.
+     */
+    protected static String mailchimpEndpoint = "https://{0}.api.mailchimp.com/3.0";
+    private final MailChimpOutputPluginDelegate.PluginTask task;
+    private final ObjectMapper mapper;
+    private final Schema schema;
+    private int requestCount;
+    private long totalCount;
+    private List<JsonNode> records;
+    private Map<String, Map<String, InterestResponse>> categories;
+
+    /**
+     * Instantiates a new Mail chimp abstract record buffer.
+     *
+     * @param schema the schema
+     * @param task   the task
+     */
+    public MailChimpAbstractRecordBuffer(final Schema schema, final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        this.schema = schema;
+        this.task = task;
+        this.mapper = new ObjectMapper()
+                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+        this.records = new ArrayList<>();
+        this.categories = new HashMap<>();
+    }
+
+    @Override
+    public void bufferRecord(ServiceRecord serviceRecord)
+    {
+        JacksonServiceRecord jacksonServiceRecord;
+
+        try {
+            jacksonServiceRecord = (JacksonServiceRecord) serviceRecord;
+            JsonNode record = mapper.readTree(jacksonServiceRecord.toString()).get("record");
+
+            requestCount++;
+            totalCount++;
+
+            records.add(record);
+            if (requestCount >= MAX_RECORD_PER_BATCH_REQUEST) {
+                ObjectNode subcribers = processSubcribers(records, task);
+                ReportResponse reportResponse = push(subcribers, task);
+
+                if (totalCount % 1000 == 0) {
+                    LOG.info("Pushed {} records", totalCount);
+                }
+
+                LOG.info("{} records created, {} records updated, {} records failed",
+                         reportResponse.getTotalCreated(),
+                         reportResponse.getTotalUpdated(),
+                         reportResponse.getErrorCount());
+                handleErrors(reportResponse.getErrors());
+
+                records = new ArrayList<>();
+                requestCount = 0;
+            }
+        }
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
+        }
+        catch (ClassCastException ex) {
+            throw new RuntimeException(ex);
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    @Override
+    public TaskReport commitWithTaskReportUpdated(TaskReport taskReport)
+    {
+        try {
+            if (records.size() > 0) {
+                ObjectNode subcribers = processSubcribers(records, task);
+                ReportResponse reportResponse = push(subcribers, task);
+                LOG.info("Pushed {} records", records.size());
+                LOG.info("{} records created, {} records updated, {} records failed",
+                         reportResponse.getTotalCreated(),
+                         reportResponse.getTotalUpdated(),
+                         reportResponse.getErrorCount());
+                handleErrors(reportResponse.getErrors());
+            }
+
+            cleanUp();
+            return Exec.newTaskReport().set("pushed", totalCount);
+        }
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
+        }
+    }
+
+    /**
+     * Receive data and build payload json that contains subscribers
+     *
+     * @param data the data
+     * @param task the task
+     * @return the object node
+     */
+    ObjectNode processSubcribers(final List<JsonNode> data, final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        LOG.info("Start to process subscriber data");
+        extractDataCenterBasedOnAuthMethod();
+        extractInterestCategories();
+
+        // Required merge fields
+        Map<String, String> map = new HashMap<>();
+        map.put("FNAME", task.getFnameColumn());
+        map.put("LNAME", task.getLnameColumn());
+
+        List<JsonNode> subscribersList = FluentIterable.from(data)
+                .transform(contactMapper(map))
+                .toList();
+
+        ObjectNode subscribers = JsonNodeFactory.instance.objectNode();
+        subscribers.putArray("members").addAll(subscribersList);
+        subscribers.put("update_existing", task.getUpdateExisting());
+        return subscribers;
+    }
+
+    /**
+     * Gets mapper.
+     *
+     * @return the mapper
+     */
+    public ObjectMapper getMapper()
+    {
+        return mapper;
+    }
+
+    /**
+     * Gets categories.
+     *
+     * @return the categories
+     */
+    public Map<String, Map<String, InterestResponse>> getCategories()
+    {
+        return categories;
+    }
+
+    /**
+     * Clean up.
+     */
+    abstract void cleanUp();
+
+    /**
+     * Push payload data to MailChimp API and get @{@link ReportResponse}
+     *
+     * @param node the content
+     * @param task the task
+     * @return the report response
+     * @throws JsonProcessingException the json processing exception
+     */
+    abstract ReportResponse push(final ObjectNode node, final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException;
+
+    /**
+     * Handle @{@link ErrorResponse} from MailChimp API if exists.
+     *
+     * @param errorResponses the error responses
+     */
+    abstract void handleErrors(final List<ErrorResponse> errorResponses);
+
+    /**
+     * Find interest category ids by pre-defined group name which user input.
+     *
+     * @param task the task
+     * @return the map
+     * @throws JsonProcessingException the json processing exception
+     */
+    abstract Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames(final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException;
+
+    /**
+     * Extract data center from MailChimp v3 metadata.
+     *
+     * @param task the task
+     * @return the string
+     * @throws JsonProcessingException the json processing exception
+     */
+    abstract String extractDataCenter(final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException;
+
+    private void extractDataCenterBasedOnAuthMethod()
+    {
+        try {
+            // Extract data center from meta data URL
+            String dc = extractDataCenter(task);
+            mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, dc);
+        }
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
+        }
+    }
+
+    private void extractInterestCategories()
+    {
+        try {
+            // Should loop the names and get the id of interest categories.
+            // The reason why we put categories validation here because we can not share data between instance.
+            categories = extractInterestCategoriesByGroupNames(task);
+        }
+        catch (JsonProcessingException jpe) {
+            throw new DataException(jpe);
+        }
+    }
+
+    private Function<JsonNode, JsonNode> contactMapper(final Map<String, String> allowColumns)
+    {
+        return new Function<JsonNode, JsonNode>()
+        {
+            @Override
+            public JsonNode apply(JsonNode input)
+            {
+                ObjectNode property = JsonNodeFactory.instance.objectNode();
+                property.put("email_address", input.findPath(task.getEmailColumn()).asText());
+                property.put("status", task.getDoubleOptIn() ? PENDING.getType() : SUBSCRIBED.getType());
+                ObjectNode mergeFields = JsonNodeFactory.instance.objectNode();
+                for (String allowColumn : allowColumns.keySet()) {
+                    String value = input.findValue(allowColumns.get(allowColumn)).asText();
+                    mergeFields.put(allowColumn, value);
+                }
+
+                // Update additional merge fields if exist
+                if (task.getMergeFields().isPresent() && !task.getMergeFields().get().isEmpty()) {
+                    for (final Column column : schema.getColumns()) {
+                        if (!"".equals(containsCaseInsensitive(column.getName(), task.getMergeFields().get()))) {
+                            String value = input.findValue(column.getName()).asText();
+                            mergeFields.put(column.getName().toUpperCase(), value);
+                        }
+                    }
+                }
+
+                property.set("merge_fields", mergeFields);
+
+                // Update interest categories if exist
+                if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
+                    property.set("interests", buildInterestCategories(task, input));
+                }
+
+                return property;
+            }
+        };
+    }
+
+    private ObjectNode buildInterestCategories(final MailChimpOutputPluginDelegate.PluginTask task,
+                                               final JsonNode input)
+    {
+        ObjectNode interests = JsonNodeFactory.instance.objectNode();
+
+        for (String category : task.getGroupingColumns().get()) {
+            String value = input.findValue(category).asText();
+            Map<String, InterestResponse> availableCategories = categories.get(category);
+
+            // Only update user-predefined categories if replace interests != true
+            // Otherwise, force update all categories include user-predefined categories
+            if (!task.getReplaceInterests() && availableCategories.get(value) != null) {
+                interests.put(availableCategories.get(value).getId(), true);
+            }
+            else if (task.getReplaceInterests()) {
+                for (String availableCategory : availableCategories.keySet()) {
+                    if (availableCategory.equals(value)) {
+                        interests.put(availableCategories.get(availableCategory).getId(), true);
+                    }
+                    else {
+                        interests.put(availableCategories.get(availableCategory).getId(), false);
+                    }
+                }
+            }
+        }
+
+        return interests;
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java

@@ -0,0 +1,151 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.MissingNode;
+import com.google.common.base.Throwables;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
+import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
+import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * Created by thangnc on 4/14/17.
+ */
+public class MailChimpHttpClient
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpHttpClient.class);
+    private final ObjectMapper jsonMapper = new ObjectMapper()
+            .configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false)
+            .configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    private Jetty92RetryHelper retryHelper;
+
+    /**
+     * Instantiates a new Mailchimp http client.
+     *
+     * @param task the task
+     */
+    public MailChimpHttpClient(MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        retryHelper = createRetryHelper(task);
+    }
+
+    /**
+     * Close @{@link Jetty92RetryHelper} connection
+     */
+    public void close()
+    {
+        if (retryHelper != null) {
+            retryHelper.close();
+        }
+    }
+
+    public JsonNode sendRequest(final String endpoint, final HttpMethod method,
+                                final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        return sendRequest(endpoint, method, "", task);
+    }
+
+    public JsonNode sendRequest(final String endpoint, final HttpMethod method, final String content,
+                                final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        final String authorizationHeader = getAuthorizationHeader(task);
+
+        String responseBody = retryHelper.requestWithRetry(
+                new StringJetty92ResponseEntityReader(task.getTimeoutMillis()),
+                new Jetty92SingleRequester()
+                {
+                    @Override
+                    public void requestOnce(HttpClient client, Response.Listener responseListener)
+                    {
+                        Request request = client
+                                .newRequest(endpoint)
+                                .accept("application/json")
+                                .method(method);
+                        if (method == HttpMethod.POST || method == HttpMethod.PUT) {
+                            request.content(new StringContentProvider(content), "application/json;utf-8");
+                        }
+
+                        if (!authorizationHeader.isEmpty()) {
+                            request.header("Authorization", authorizationHeader);
+                        }
+                        request.send(responseListener);
+                    }
+
+                    @Override
+                    public boolean isResponseStatusToRetry(Response response)
+                    {
+                        int status = response.getStatus();
+                        return status == 429 || status / 100 != 4;
+                    }
+                });
+
+        return responseBody != null && !responseBody.isEmpty() ? parseJson(responseBody) : MissingNode.getInstance();
+    }
+
+    private JsonNode parseJson(final String json)
+            throws DataException
+    {
+        try {
+            return this.jsonMapper.readTree(json);
+        }
+        catch (IOException ex) {
+            throw new DataException(ex);
+        }
+    }
+
+    /**
+     * MailChimp API v3 supports non expires access_token. Then no need refresh_token
+     *
+     * @param task
+     * @return
+     */
+    private String getAuthorizationHeader(final MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        switch (task.getAuthMethod()) {
+            case OAUTH:
+                return "OAuth " + task.getAccessToken().orNull();
+            case API_KEY:
+                return "Basic " + Base64Variants.MIME_NO_LINEFEEDS
+                        .encode(("apikey" + ":" + task.getApikey().orNull()).getBytes());
+            default:
+                throw new ConfigException("Not supported method");
+        }
+    }
+
+    private Jetty92RetryHelper createRetryHelper(MailChimpOutputPluginDelegate.PluginTask task)
+    {
+        return new Jetty92RetryHelper(
+                task.getMaximumRetries(),
+                task.getInitialRetryIntervalMillis(),
+                task.getMaximumRetryIntervalMillis(),
+                new Jetty92ClientCreator()
+                {
+                    @Override
+                    public HttpClient createAndStart()
+                    {
+                        HttpClient client = new HttpClient(new SslContextFactory());
+                        try {
+                            client.start();
+                            return client;
+                        }
+                        catch (Exception e) {
+                            throw Throwables.propagate(e);
+                        }
+                    }
+                });
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPlugin.java

@@ -0,0 +1,18 @@
+package org.embulk.output.mailchimp;
+
+import org.embulk.base.restclient.RestClientOutputPluginBase;
+
+/**
+ * Created by thangnc on 4/14/17.
+ */
+public class MailChimpOutputPlugin
+        extends RestClientOutputPluginBase<MailChimpOutputPluginDelegate.PluginTask>
+{
+    /**
+     * Instantiates a new @{@link MailChimpOutputPlugin}.
+     */
+    public MailChimpOutputPlugin()
+    {
+        super(MailChimpOutputPluginDelegate.PluginTask.class, new MailChimpOutputPluginDelegate());
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java

@@ -0,0 +1,164 @@
+package org.embulk.output.mailchimp;
+
+import com.google.common.base.Optional;
+import org.embulk.base.restclient.RestClientOutputPluginDelegate;
+import org.embulk.base.restclient.RestClientOutputTaskBase;
+import org.embulk.base.restclient.jackson.JacksonServiceRequestMapper;
+import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
+import org.embulk.base.restclient.jackson.scope.JacksonAllInObjectScope;
+import org.embulk.base.restclient.record.RecordBuffer;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.output.mailchimp.model.AuthMethod;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+
+import java.util.List;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.embulk.output.mailchimp.validation.ColumnDataValidator.checkExistColumns;
+
+/**
+ * Created by thangnc on 4/14/17.
+ */
+public class MailChimpOutputPluginDelegate
+        implements RestClientOutputPluginDelegate<MailChimpOutputPluginDelegate.PluginTask>
+{
+    private static final Logger LOG = Exec.getLogger(MailChimpOutputPluginDelegate.class);
+
+    public MailChimpOutputPluginDelegate()
+    {
+    }
+
+    public interface PluginTask
+            extends RestClientOutputTaskBase
+    {
+        @Config("maximum_retries")
+        @ConfigDefault("6")
+        int getMaximumRetries();
+
+        @Config("initial_retry_interval_millis")
+        @ConfigDefault("1000")
+        int getInitialRetryIntervalMillis();
+
+        @Config("maximum_retry_interval_millis")
+        @ConfigDefault("32000")
+        int getMaximumRetryIntervalMillis();
+
+        @Config("timeout_millis")
+        @ConfigDefault("60000")
+        int getTimeoutMillis();
+
+        @Config("auth_method")
+        @ConfigDefault("api_key")
+        AuthMethod getAuthMethod();
+
+        @Config("apikey")
+        @ConfigDefault("null")
+        Optional<String> getApikey();
+
+        @Config("access_token")
+        @ConfigDefault("null")
+        Optional<String> getAccessToken();
+
+        @Config("list_id")
+        String getListId();
+
+        @Config("email_column")
+        @ConfigDefault("email")
+        String getEmailColumn();
+
+        @Config("fname_column")
+        @ConfigDefault("fname")
+        String getFnameColumn();
+
+        @Config("lname_column")
+        @ConfigDefault("lname")
+        String getLnameColumn();
+
+        @Config("merge_fields")
+        @ConfigDefault("null")
+        Optional<List<String>> getMergeFields();
+
+        @Config("grouping_columns")
+        @ConfigDefault("null")
+        Optional<List<String>> getGroupingColumns();
+
+        @Config("double_optin")
+        @ConfigDefault("true")
+        boolean getDoubleOptIn();
+
+        @Config("update_existing")
+        @ConfigDefault("false")
+        boolean getUpdateExisting();
+
+        @Config("replace_interests")
+        @ConfigDefault("true")
+        boolean getReplaceInterests();
+    }
+
+    /**
+     * Override @{@link RestClientOutputPluginDelegate#validateOutputTask(RestClientOutputTaskBase, Schema, int)}
+     * This method not only validates required configurations but also validates required columns
+     *
+     * @param task
+     * @param schema
+     * @param taskCount
+     */
+    @Override
+    public void validateOutputTask(final PluginTask task, final Schema schema, final int taskCount)
+    {
+        if (task.getAuthMethod() == AuthMethod.OAUTH) {
+            if (!task.getAccessToken().isPresent() || isNullOrEmpty(task.getAccessToken().get())) {
+                throw new ConfigException("'access_token' is required when auth_method is 'oauth'");
+            }
+        }
+        else if (task.getAuthMethod() == AuthMethod.API_KEY) {
+            if (!task.getApikey().isPresent() || isNullOrEmpty(task.getApikey().get())) {
+                throw new ConfigException("'apikey' is required when auth_method is 'api_key'");
+            }
+        }
+
+        if (isNullOrEmpty(task.getListId())) {
+            throw new ConfigException("'list_id' must not be null or empty string");
+        }
+
+        if (!checkExistColumns(schema, task.getEmailColumn(), task.getFnameColumn(), task.getLnameColumn())) {
+            throw new ConfigException("Columns ['email', 'fname', 'lname'] must not be null or empty string");
+        }
+    }
+
+    @Override
+    public RecordBuffer buildRecordBuffer(PluginTask task, Schema schema, int taskIndex)
+    {
+        return new MailChimpRecordBuffer(schema, task);
+    }
+
+    @Override
+    public JacksonServiceRequestMapper buildServiceRequestMapper(final PluginTask task)
+    {
+        return JacksonServiceRequestMapper.builder()
+                .add(new JacksonAllInObjectScope(), new JacksonTopLevelValueLocator("record"))
+                .build();
+    }
+
+    @Override
+    public ConfigDiff egestEmbulkData(final PluginTask task, final Schema schema, final int taskCount,
+                                      final List<TaskReport> taskReports)
+    {
+        long totalInserted = 0;
+        for (TaskReport taskReport : taskReports) {
+            if (taskReport.has("pushed")) {
+                totalInserted += taskReport.get(Long.class, "pushed");
+            }
+        }
+
+        LOG.info("Pushed completed. {} records", totalInserted);
+
+        return Exec.newConfigDiff();
+    }
+}