embulk-output-mailchimp 0.3.25 → 0.3.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94b54333acb168038a0b2537d6bf7c8a71be4993
-  data.tar.gz: 18b422345048fd9d9681dc24a369c8c16680c55e
+  metadata.gz: b429373deb2a79bd444d223f70818c7a7517bd63
+  data.tar.gz: 06fb08e83602804217e386754e39cd7cb542c2e8
 SHA512:
-  metadata.gz: 8248823129afda3c947fed964e9ea6d080cf7cfb81b18d399cf9ab4aae25b40ead8e71a0981a98a9c184b018b5cc7c948b1df5aef639b35f7c03dba81ce6d7fc
-  data.tar.gz: 24220688ee272036fa55ce8cf040ee3c95ce76942e424a35121767bf4f8e7924985b236712ffeac1d57ef75c2ae3441ff8646e67cf63b55d5e963c5e0ec97b56
+  metadata.gz: b096372c93fcb6a0c2930d0b98ade33a1d52277f65fd1554e6d80178ee2179b300847c69c907d4e2cbb1279f7028e6eac3804d123f8e92681e1955182df5bc22
+  data.tar.gz: efd31474670848d2457b2fdc5ea68b990902971543e884dd473fcb57064b34687a79beb95188bc06349e5448cac1a4919b91cadc90083a0fe39cca32d09c5c60

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.28 - 2018-10-29
+
+- Quick fix a typo
+
+## 0.3.26 - 2018-10-29
+
+- Fix an NPE when there isn't a column corresponds to a configured group ('category')
+
 ## 0.3.25 - 2018-10-11
 
 - Fixed an NPE when column names and merge tags are not exactly (case-sensitive) matched

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.25"
+version = "0.3.28"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
+import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.FluentIterable;
 import org.embulk.base.restclient.jackson.JacksonServiceRecord;
@@ -25,6 +26,8 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -34,11 +37,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
+import static com.google.common.base.Joiner.on;
 import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import static java.lang.String.format;
 import static org.embulk.output.mailchimp.MailChimpOutputPluginDelegate.PluginTask;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.fromCommaSeparatedString;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.jsonGetIgnoreCase;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.orderJsonNode;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
 import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
@@ -168,6 +174,13 @@ public class MailChimpRecordBuffer
         // The reason why we put categories validation here because we can not share data between instance.
         if (categories == null) {
             categories = mailChimpClient.extractInterestCategoriesByGroupNames(task);
+
+            Set<String> categoriesNames = categories.keySet();
+            Set<String> columnNames = caseInsensitiveColumnNames();
+            if (!columnNames.containsAll(categoriesNames)) {
+                categoriesNames.removeAll(columnNames);
+                LOG.warn("Data column for category '{}' could not be found", on(", ").join(categoriesNames));
+            }
         }
 
         // Extract merge fields detail
@@ -249,7 +262,7 @@ public class MailChimpRecordBuffer
 
                 // Update interest categories if exist
                 if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
-                    property.set("interests", buildInterestCategories(task, input));
+                    property.set("interests", buildInterestCategories(input));
                 }
 
                 // Update language if exist
@@ -262,31 +275,36 @@ public class MailChimpRecordBuffer
         };
     }
 
-    private ObjectNode buildInterestCategories(final PluginTask task, final JsonNode input)
+    private ObjectNode buildInterestCategories(final JsonNode input)
     {
         ObjectNode interests = JsonNodeFactory.instance.objectNode();
 
         if (task.getGroupingColumns().isPresent()) {
             for (String category : task.getGroupingColumns().get()) {
-                String inputValue = input.findValue(category).asText();
-                List<String> interestValues = fromCommaSeparatedString(inputValue);
-                Map<String, InterestResponse> availableCategories = categories.get(category);
+                Optional<JsonNode> inputValue = jsonGetIgnoreCase(input, category);
+                if (!inputValue.isPresent()) {
+                    // Silently ignore if the grouping column is absent
+                    continue;
+                }
+                List<String> recordInterests = fromCommaSeparatedString(inputValue.get().asText());
+                // `categories` is guaranteed to contain the `category` as it already did an early check
+                Map<String, InterestResponse> availableInterests = categories.get(category);
 
                 // Only update user-predefined categories if replace interests != true
                 if (!task.getReplaceInterests()) {
-                    for (String interestValue : interestValues) {
-                        if (availableCategories.get(interestValue) != null) {
-                            interests.put(availableCategories.get(interestValue).getId(), true);
+                    for (String recordInterest : recordInterests) {
+                        if (availableInterests.get(recordInterest) != null) {
+                            interests.put(availableInterests.get(recordInterest).getId(), true);
                         }
                     }
                 } // Otherwise, force update all categories include user-predefined categories
                 else if (task.getReplaceInterests()) {
-                    for (String availableCategory : availableCategories.keySet()) {
-                        if (interestValues.contains(availableCategory)) {
-                            interests.put(availableCategories.get(availableCategory).getId(), true);
+                    for (String availableInterest : availableInterests.keySet()) {
+                        if (recordInterests.contains(availableInterest)) {
+                            interests.put(availableInterests.get(availableInterest).getId(), true);
                         }
                         else {
-                            interests.put(availableCategories.get(availableCategory).getId(), false);
+                            interests.put(availableInterests.get(availableInterest).getId(), false);
                         }
                     }
                 }
@@ -313,7 +331,8 @@ public class MailChimpRecordBuffer
     private void pushData() throws JsonProcessingException
     {
         long startTime = System.currentTimeMillis();
-        ObjectNode subscribers = processSubcribers(uniqueRecords, task);
+        ObjectNode subscribers =
+                processSubcribers(uniqueRecords, task);
         ReportResponse reportResponse = mailChimpClient.push(subscribers, task);
 
         LOG.info("Done with {} record(s). Response from MailChimp: {} records created, {} records updated, {} records failed. Batch took {} ms ",
@@ -340,4 +359,21 @@ public class MailChimpRecordBuffer
             }
         }
     }
+
+    private Set<String> caseInsensitiveColumnNames()
+    {
+        Set<String> columns = new TreeSet<>(CASE_INSENSITIVE_ORDER);
+        columns.addAll(FluentIterable
+                .from(schema.getColumns())
+                .transform(new Function<Column, String>() {
+                    @Nullable
+                    @Override
+                    public String apply(@Nullable Column col)
+                    {
+                        return col.getName();
+                    }
+                })
+                .toSet());
+        return columns;
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java

@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
+import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -16,8 +17,11 @@ import org.embulk.output.mailchimp.model.AddressMergeFieldAttribute;
 import javax.annotation.Nullable;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Created by thangnc on 4/26/17.
  */
@@ -125,4 +129,17 @@ public final class MailChimpHelper
 
         return orderedNode;
     }
+
+    public static Optional<JsonNode> jsonGetIgnoreCase(JsonNode node, String fieldName)
+    {
+        requireNonNull(fieldName);
+        Iterator<String> fieldNames = node.fieldNames();
+        while (fieldNames.hasNext()) {
+            String curFieldName = fieldNames.next();
+            if (fieldName.equalsIgnoreCase((curFieldName))) {
+                return Optional.of(node.get(curFieldName));
+            }
+        }
+        return Optional.absent();
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpRetryable.java

@@ -24,6 +24,7 @@ import java.text.MessageFormat;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
 
+import static com.google.common.base.Throwables.propagate;
 import static org.eclipse.jetty.http.HttpHeader.AUTHORIZATION;
 import static org.eclipse.jetty.http.HttpMethod.GET;
 import static org.eclipse.jetty.http.HttpMethod.POST;
@@ -64,41 +65,47 @@ public class MailChimpRetryable implements AutoCloseable
 
     private String sendRequest(final String path, final StringContentProvider contentProvider)
     {
-        return retryHelper.requestWithRetry(
-                new StringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
-                new Jetty92SingleRequester()
-                {
-                    @Override
-                    public void requestOnce(HttpClient client, Response.Listener responseListener)
+        try {
+            return retryHelper.requestWithRetry(
+                    new StringJetty92ResponseEntityReader(READER_TIMEOUT_MILLIS),
+                    new Jetty92SingleRequester()
                     {
-                        createTokenHolder(client);
-                        Request request = client.newRequest(tokenHolder.getEndpoint() + path)
-                                .header(AUTHORIZATION, authorizationHeader)
-                                .method(GET);
-                        if (contentProvider != null) {
-                            request = request.method(POST).content(contentProvider);
+                        @Override
+                        public void requestOnce(HttpClient client, Response.Listener responseListener)
+                        {
+                            createTokenHolder(client);
+                            Request request = client.newRequest(tokenHolder.getEndpoint() + path)
+                                    .header(AUTHORIZATION, authorizationHeader)
+                                    .method(GET);
+                            if (contentProvider != null) {
+                                request = request.method(POST).content(contentProvider);
+                            }
+                            request.send(responseListener);
                         }
-                        request.send(responseListener);
-                    }
 
-                    @Override
-                    protected boolean isResponseStatusToRetry(Response response)
-                    {
-                        // Retry if it's a server or rate limit exceeded error
-                        return (response.getStatus() != 500 && response.getStatus() / 100 != 4) || response.getStatus() == 429;
-                    }
+                        @Override
+                        protected boolean isResponseStatusToRetry(Response response)
+                        {
+                            // Retry if it's a server or rate limit exceeded error
+                            return (response.getStatus() != 500 && response.getStatus() / 100 != 4) || response.getStatus() == 429;
+                        }
 
-                    @Override
-                    protected boolean isExceptionToRetry(Exception exception)
-                    {
-                        // This check is to make sure if the original exception is retryable, i.e.
-                        // server not found, internal server error...
-                        if (exception instanceof ConfigException || exception instanceof ExecutionException) {
-                            return toRetry((Exception) exception.getCause());
+                        @Override
+                        protected boolean isExceptionToRetry(Exception exception)
+                        {
+                            // This check is to make sure if the original exception is retryable, i.e.
+                            // server not found, internal server error...
+                            if (exception instanceof ConfigException || exception instanceof ExecutionException) {
+                                return toRetry((Exception) exception.getCause());
+                            }
+                            return exception instanceof TimeoutException || super.isExceptionToRetry(exception);
                         }
-                        return exception instanceof TimeoutException || super.isExceptionToRetry(exception);
-                    }
-                });
+                    });
+        }
+        catch (HttpResponseException ex) {
+            LOG.error("Unexpected response from request to {}", path, ex);
+            throw propagate(ex);
+        }
     }
 
     @Override

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.25
+  version: 0.3.28
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-11 00:00:00.000000000 Z
+date: 2018-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -89,12 +89,12 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/jetty-io-9.2.14.v20151106.jar
-- classpath/embulk-output-mailchimp-0.3.25.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
+- classpath/embulk-output-mailchimp-0.3.28.jar
 homepage: https://github.com/treasure-data/embulk-output-mailchimp
 licenses:
 - MIT