embulk-output-mailchimp 0.3.4 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 743479cc26e1bed837531a34c605df1f27ae1ee8
-  data.tar.gz: 5d6af1f50147f6a79447efd2ba118862088b959d
+  metadata.gz: a306f852f0eea3bd2d95ef46dcff661f63c5f3bf
+  data.tar.gz: e6be5cde31d3ade89061b961cbc7bc0fd80ac257
 SHA512:
-  metadata.gz: 7c79489c197c389155db2a6ef243b78152bc9f51069d81d88c3dd13502d55b254c1de6e6519e93f597619422897888914bad66778cdda4e3b50ac8ebb8ab1183
-  data.tar.gz: ccc953a7406261e40cacc0881b4bafe81564381b718781588b870f3502a738d59df19a6b59f31f14253598d62807177c09ccca29c55f85ecd78e9624d3a406bd
+  metadata.gz: d0daa9b8299e3858e6c5347f1ec72fafb580eb88b61ffd905dc8180e5179b5aec27925804f7f3ade25a02b5fc23ffdf5900d4b771f55a393254be5e13e2d73de
+  data.tar.gz: d3b0c68f30be686c2fbd092d6e14ec455562ee9d7da198e319c7bfcb68a9c8457a0a672533ba7d0195ed90951ef0f4f6df0721074d073e5b206a068f72214806

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.6
+- Removed log query and add API to check address type [#22](https://github.com/treasure-data/embulk-output-mailchimp/pull/22)
+- Enabled log query for logging on console temporary [#21](https://github.com/treasure-data/embulk-output-mailchimp/pull/21)
+
 ## 0.3.4 - 2017-06-01
 - Enable JSON type for `address` MERGE field [#20](https://github.com/treasure-data/embulk-output-mailchimp/pull/20)
 
@@ -8,19 +12,11 @@
 - Rename `interest_categories` to `grouping_columns` to fix backward compatibility [#18](https://github.com/treasure-data/embulk-output-mailchimp/pull/18)
 
 ## 0.3.1 - 2017-05-26
-- Enable merge fields with case-insensitive [#17](https://github.com/treasure-data/embulk-output-mailchimp/pull/17)
-
-## 0.3.0.4 - 2017-05-24
-- Fixed compatible with API v2 and changed data format [#16](https://github.com/treasure-data/embulk-output-mailchimp/pull/16)
-
-## 0.3.0.3 - 2017-05-18
-- Fixed bug can not extract data center when use `api_key` as `auth_mode` [#15](https://github.com/treasure-data/embulk-output-mailchimp/pull/15)
-
-## 0.3.0.2 - 2017-05-18
-- Enabled double_optin in configuration and use default status if schema has no column `status` [#15](https://github.com/treasure-data/embulk-output-mailchimp/pull/15)
-
-## 0.3.0.1 - 2017-05-11
 - Enabled API v3 and supported OAuth2 beside API Key[#13](https://github.com/treasure-data/embulk-output-mailchimp/pull/13)
+- Enabled double_optin in configuration and use default status if schema has no column `status` [#15](https://github.com/treasure-data/embulk-output-mailchimp/pull/15)
+- Fixed bug can not extract data center when use `api_key` as `auth_mode` [#15](https://github.com/treasure-data/embulk-output-mailchimp/pull/15)
+- Fixed compatible with API v2 and changed data format [#16](https://github.com/treasure-data/embulk-output-mailchimp/pull/16)
+- Enable merge fields with case-insensitive [#17](https://github.com/treasure-data/embulk-output-mailchimp/pull/17)
 
 ## 0.2.3 - 2016-03-24
 

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.4"
+version = "0.3.6"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpAbstractRecordBuffer.java

@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
@@ -16,6 +17,7 @@ import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.config.TaskReport;
 import org.embulk.output.mailchimp.model.ErrorResponse;
 import org.embulk.output.mailchimp.model.InterestResponse;
+import org.embulk.output.mailchimp.model.MergeField;
 import org.embulk.output.mailchimp.model.ReportResponse;
 import org.embulk.spi.Column;
 import org.embulk.spi.DataException;
@@ -32,9 +34,9 @@ import java.util.Map;
 
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.fromCommaSeparatedString;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
 import static org.embulk.output.mailchimp.model.MemberStatus.PENDING;
 import static org.embulk.output.mailchimp.model.MemberStatus.SUBSCRIBED;
-import static org.embulk.spi.type.Types.JSON;
 
 /**
  * Created by thangnc on 4/14/17.
@@ -55,6 +57,7 @@ public abstract class MailChimpAbstractRecordBuffer
     private long totalCount;
     private List<JsonNode> records;
     private Map<String, Map<String, InterestResponse>> categories;
+    private Map<String, MergeField> availableMergeFields;
 
     /**
      * Instantiates a new Mail chimp abstract record buffer.
@@ -146,10 +149,19 @@ public abstract class MailChimpAbstractRecordBuffer
      * @return the object node
      */
     ObjectNode processSubcribers(final List<JsonNode> data, final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException
     {
         LOG.info("Start to process subscriber data");
-        extractDataCenterBasedOnAuthMethod();
-        extractInterestCategories();
+        // Extract data center from meta data URL
+        String dc = extractDataCenter(task);
+        mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, dc);
+
+        // Should loop the names and get the id of interest categories.
+        // The reason why we put categories validation here because we can not share data between instance.
+        categories = extractInterestCategoriesByGroupNames(task);
+
+        // Extract merge fields detail
+        availableMergeFields = extractMergeFieldsFromList(task);
 
         // Required merge fields
         Map<String, String> map = new HashMap<>();
@@ -229,29 +241,15 @@ public abstract class MailChimpAbstractRecordBuffer
     abstract String extractDataCenter(final MailChimpOutputPluginDelegate.PluginTask task)
             throws JsonProcessingException;
 
-    private void extractDataCenterBasedOnAuthMethod()
-    {
-        try {
-            // Extract data center from meta data URL
-            String dc = extractDataCenter(task);
-            mailchimpEndpoint = MessageFormat.format(mailchimpEndpoint, dc);
-        }
-        catch (JsonProcessingException jpe) {
-            throw new DataException(jpe);
-        }
-    }
-
-    private void extractInterestCategories()
-    {
-        try {
-            // Should loop the names and get the id of interest categories.
-            // The reason why we put categories validation here because we can not share data between instance.
-            categories = extractInterestCategoriesByGroupNames(task);
-        }
-        catch (JsonProcessingException jpe) {
-            throw new DataException(jpe);
-        }
-    }
+    /**
+     * Extract all merge fields from MailChimp list.
+     *
+     * @param task the task
+     * @return the map
+     * @throws JsonProcessingException the json processing exception
+     */
+    abstract Map<String, MergeField> extractMergeFieldsFromList(final MailChimpOutputPluginDelegate.PluginTask task)
+            throws JsonProcessingException;
 
     private Function<JsonNode, JsonNode> contactMapper(final Map<String, String> allowColumns)
     {
@@ -273,11 +271,20 @@ public abstract class MailChimpAbstractRecordBuffer
                 if (task.getMergeFields().isPresent() && !task.getMergeFields().get().isEmpty()) {
                     for (final Column column : schema.getColumns()) {
                         if (!"".equals(containsCaseInsensitive(column.getName(), task.getMergeFields().get()))) {
-                            if (column.getType().equals(JSON)) {
-                                mergeFields.set(column.getName().toUpperCase(), input.findValue(column.getName()));
+                            String value = input.findValue(column.getName()).asText();
+
+                            // Try to convert to Json from string with the merge field's type is address
+                            if (availableMergeFields.get(column.getName()).getType()
+                                    .equals(MergeField.MergeFieldType.ADDRESS.getType())) {
+                                JsonNode addressNode = toJsonNode(value);
+                                if (addressNode instanceof NullNode) {
+                                    mergeFields.put(column.getName().toUpperCase(), value);
+                                }
+                                else {
+                                    mergeFields.set(column.getName().toUpperCase(), addressNode);
+                                }
                             }
                             else {
-                                String value = input.findValue(column.getName()).asText();
                                 mergeFields.put(column.getName().toUpperCase(), value);
                             }
                         }

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -15,6 +15,8 @@ import org.embulk.output.mailchimp.model.ErrorResponse;
 import org.embulk.output.mailchimp.model.InterestCategoriesResponse;
 import org.embulk.output.mailchimp.model.InterestResponse;
 import org.embulk.output.mailchimp.model.InterestsResponse;
+import org.embulk.output.mailchimp.model.MergeField;
+import org.embulk.output.mailchimp.model.MergeFields;
 import org.embulk.output.mailchimp.model.MetaDataResponse;
 import org.embulk.output.mailchimp.model.ReportResponse;
 import org.embulk.spi.Exec;
@@ -156,6 +158,17 @@ public class MailChimpRecordBuffer extends MailChimpAbstractRecordBuffer
         }
     }
 
+    @Override
+    Map<String, MergeField> extractMergeFieldsFromList(MailChimpOutputPluginDelegate.PluginTask task) throws JsonProcessingException
+    {
+        String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/merge-fields",
+                                               task.getListId());
+        JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
+        MergeFields mergeFields = getMapper().treeToValue(response,
+                                                          MergeFields.class);
+        return convertMergeFieldToMap(mergeFields.getMergeFields());
+    }
+
     private Map<String, InterestResponse> convertInterestCategoryToMap(final List<InterestResponse> interestResponseList)
     {
         Function<InterestResponse, String> function = new Function<InterestResponse, String>()
@@ -171,4 +184,21 @@ public class MailChimpRecordBuffer extends MailChimpAbstractRecordBuffer
                                         .toList(),
                                 function);
     }
+
+    private Map<String, MergeField> convertMergeFieldToMap(final List<MergeField> mergeFieldList)
+    {
+        Function<MergeField, String> function = new Function<MergeField, String>()
+        {
+            @Nullable
+            @Override
+            public String apply(@Nullable MergeField input)
+            {
+                return input.getTag().toLowerCase();
+            }
+        };
+
+        return Maps.uniqueIndex(FluentIterable.from(mergeFieldList)
+                                        .toList(),
+                                function);
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java

@@ -1,6 +1,10 @@
 package org.embulk.output.mailchimp.helper;
 
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.google.common.base.Function;
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
@@ -9,6 +13,7 @@ import com.google.common.collect.Multimaps;
 
 import javax.annotation.Nullable;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -76,9 +81,28 @@ public final class MailChimpHelper
      * @param string the string
      * @return the list
      */
-    public static List<String> fromCommaSeparatedString(String string)
+    public static List<String> fromCommaSeparatedString(final String string)
     {
         Iterable<String> split = Splitter.on(",").omitEmptyStrings().trimResults().split(string);
         return Lists.newArrayList(split);
     }
+
+    /**
+     * TODO: td-worker automatically converts Presto json type to Embulk string type. This is wordaround to convert String to JsonNode
+     *
+     * @param string the string
+     * @return the json node
+     */
+    public static JsonNode toJsonNode(final String string)
+    {
+        final ObjectMapper mapper = new ObjectMapper()
+                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+        try {
+            return mapper.readTree(string);
+        }
+        catch (IOException e) {
+            return JsonNodeFactory.instance.nullNode();
+        }
+    }
 }

data/src/main/java/org/embulk/output/mailchimp/model/MergeField.java

@@ -0,0 +1,102 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.config.ConfigException;
+
+/**
+ * Created by thangnc on 5/8/17.
+ */
+public class MergeField
+{
+    @JsonProperty("merge_id")
+    private int mergeId;
+
+    private String name;
+    private String tag;
+    private String type;
+
+    public int getMergeId()
+    {
+        return mergeId;
+    }
+
+    public void setMergeId(int mergeId)
+    {
+        this.mergeId = mergeId;
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public void setName(String name)
+    {
+        this.name = name;
+    }
+
+    public String getTag()
+    {
+        return tag;
+    }
+
+    public void setTag(String tag)
+    {
+        this.tag = tag;
+    }
+
+    public String getType()
+    {
+        return type;
+    }
+
+    public void setType(String type)
+    {
+        this.type = type;
+    }
+
+    public enum MergeFieldType
+    {
+        TEXT("text"), NUMBER("number"), ADDRESS("address"), PHONE("phone"), DATE("date"), URL("url"),
+        IMAGEURL("imageurl"), RADIO("radio"), DROPDOWN("dropdown"), BIRTHDAY("birthday"), ZIP("zip");
+
+        private String type;
+
+        MergeFieldType(String type)
+        {
+            this.type = type;
+        }
+
+        /**
+         * Gets type.
+         *
+         * @return the type
+         */
+        public String getType()
+        {
+            return type;
+        }
+
+        /**
+         * Find by type auth method.
+         *
+         * @param type the type
+         * @return the auth method
+         */
+        @JsonCreator
+        public static MergeFieldType findByType(final String type)
+        {
+            for (MergeFieldType method : values()) {
+                if (method.getType().equals(type.toLowerCase())) {
+                    return method;
+                }
+            }
+
+            throw new ConfigException(
+                    String.format("Unknown merge field type '%s'. Supported targets are [text, number, address, phone, " +
+                                          "date, url, imageurl, radio, dropdown, birthday, zip]",
+                                  type));
+        }
+    }
+}

data/src/main/java/org/embulk/output/mailchimp/model/MergeFields.java

@@ -0,0 +1,24 @@
+package org.embulk.output.mailchimp.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+/**
+ * Created by thangnc on 5/8/17.
+ */
+public class MergeFields
+{
+    @JsonProperty("merge_fields")
+    private List<MergeField> mergeFields;
+
+    public List<MergeField> getMergeFields()
+    {
+        return mergeFields;
+    }
+
+    public void setMergeFields(List<MergeField> mergeFields)
+    {
+        this.mergeFields = mergeFields;
+    }
+}

data/src/test/java/org/embulk/output/mailchimp/TestMailChimpHelper.java

@@ -2,6 +2,8 @@ package org.embulk.output.mailchimp;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Multimap;
 import org.embulk.output.mailchimp.helper.MailChimpHelper;
 import org.junit.Test;
@@ -13,6 +15,7 @@ import java.util.List;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.extractMemberStatus;
 import static org.embulk.output.mailchimp.helper.MailChimpHelper.maskEmail;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.toJsonNode;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
@@ -63,4 +66,20 @@ public class TestMailChimpHelper
         assertEquals("Length should match", expect.length, separatedString.size());
         assertArrayEquals("Should match", expect, separatedString.toArray());
     }
+
+    @Test
+    public void test_toJsonNode_validJsonString()
+    {
+        String given = "{\"addr1\":\"1234\",\"city\":\"mountain view\",\"country\":\"US\",\"state\":\"CA\",\"zip\":\"95869\"}";
+        String expect = "US";
+
+        assertEquals("Should be Json", ObjectNode.class, toJsonNode(given).getClass());
+        assertEquals("Should have attribute `country`", expect, toJsonNode(given).get("country").asText());
+    }
+
+    @Test
+    public void test_toJsonNode_invalidJSonString()
+    {
+        assertEquals("Should be NullNode", NullNode.class, toJsonNode("abc").getClass());
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.6
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-01 00:00:00.000000000 Z
+date: 2017-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +74,8 @@ files:
 - src/main/java/org/embulk/output/mailchimp/model/InterestResponse.java
 - src/main/java/org/embulk/output/mailchimp/model/InterestsResponse.java
 - src/main/java/org/embulk/output/mailchimp/model/MemberStatus.java
+- src/main/java/org/embulk/output/mailchimp/model/MergeField.java
+- src/main/java/org/embulk/output/mailchimp/model/MergeFields.java
 - src/main/java/org/embulk/output/mailchimp/model/MetaDataResponse.java
 - src/main/java/org/embulk/output/mailchimp/model/ReportResponse.java
 - src/main/java/org/embulk/output/mailchimp/validation/ColumnDataValidator.java
@@ -86,7 +88,7 @@ files:
 - test/override_assert_raise.rb
 - test/run-test.rb
 - classpath/embulk-base-restclient-0.5.0.jar
-- classpath/embulk-output-mailchimp-0.3.4.jar
+- classpath/embulk-output-mailchimp-0.3.6.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.0.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar