embulk-output-mailchimp 0.3.18 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d40d14e9983581d16ab09fb8c909b443eb25373
-  data.tar.gz: ed754d0a34cae59e5d0e6e36012a44bcac293551
+  metadata.gz: c09e4eb035f05f307279f2ecf2ce11c3050b1cf6
+  data.tar.gz: c38bcf2a895c5c87cdb8d075867fac68cda8b15c
 SHA512:
-  metadata.gz: 274b98c648edc2175e9551996e77a136dbe046dd38efa410deb94adb3c308a05a8c5ac8e0cab0c7e6f4d24771798f1ad6a51270c69108b85f6c28321e37edf4b
-  data.tar.gz: 78652eeea2f7b95488bf6fbff24b986af8dbe2ed8a3d4faf8fcf6e42401b7bdbe24eab80738de4e5e8071bad47d3df196d150c421747668bcefb311a4025028a
+  metadata.gz: fbe4afd20c07d7863ae67c5cd35617b12de3bcedea65b7cc271689ef65f7ca84a1d0f8c508bceac15531407739c220eb868afc78c35d773ce68608ca7291f68a
+  data.tar.gz: 8ade98e5dde219c3e43e2711bd50c71ff23c32c38e6510235cb44c320f09c2094832e33ea4bd4a6cd88fe041c6a0f947553468889d2ae66f7797223e315fb4e0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.19 - 2017-10-10
+- Fixed bug can not parse invalid JSON response and added request timeout to avoid flush MailChimp API [#35](https://github.com/treasure-data/embulk-output-mailchimp/pull/35)
+
 ## 0.3.18 - 2017-09-19
 - Fixed error parsing MailChimp API response JSON when push the large data [#34](https://github.com/treasure-data/embulk-output-mailchimp/pull/34)
 

data/README.md

@@ -25,6 +25,7 @@ add e-mail to List in MailChimp.
 - **grouping_columns**: Array for group names in MailChimp dashboard(array, default: nil)
 - **language_column**: column name for language (string, optional, default: nil)
 - **double_optin**: control whether to send an opt-in confirmation email (boolean, default: true)
+- **max_records_per_request**: The max records per batch request. MailChimp API enables max records is 500 per batch request (int, default: 500)
 
 ## Example
 

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.18"
+version = "0.3.19"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.MissingNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
@@ -31,6 +32,7 @@ import org.slf4j.Logger;
 import javax.annotation.Nullable;
 
 import java.text.MessageFormat;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -80,6 +82,14 @@ public class MailChimpClient
                                                task.getListId());
 
         JsonNode response = client.sendRequest(endpoint, HttpMethod.POST, node.toString(), task);
+        client.avoidFlushAPI("Pushing next request");
+
+        if (response instanceof MissingNode) {
+            ReportResponse reportResponse = new ReportResponse();
+            reportResponse.setErrors(new ArrayList<ErrorResponse>());
+            return reportResponse;
+        }
+
         return mapper.treeToValue(response, ReportResponse.class);
     }
 
@@ -153,6 +163,9 @@ public class MailChimpClient
                                                              task.getListId(),
                                                              categoriesResponse.getId());
                 response = client.sendRequest(detailEndpoint, HttpMethod.GET, task);
+
+                // Avoid flush MailChimp API
+                client.avoidFlushAPI("Fetching next category's interests");
                 InterestsResponse interestsResponse = mapper.treeToValue(response, InterestsResponse.class);
                 categories.put(categoriesResponse.getTitle().toLowerCase(),
                                convertInterestCategoryToMap(interestsResponse.getInterests()));

data/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java

@@ -24,6 +24,7 @@ import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 /**
@@ -66,6 +67,7 @@ public class MailChimpHttpClient
                         {
                             Request request = client
                                     .newRequest(endpoint)
+                                    .timeout(task.getTimeoutMillis(), TimeUnit.MILLISECONDS)
                                     .accept("application/json")
                                     .method(method);
                             if (method == HttpMethod.POST || method == HttpMethod.PUT) {
@@ -117,11 +119,47 @@ public class MailChimpHttpClient
         try {
             return this.jsonMapper.readTree(json);
         }
+        catch (IOException ex) {
+            // Try to parse invalid json before throwing exception
+            return parseInvalidJsonString(json);
+        }
+    }
+
+    // Sometimes, the MailChimp API returns invalid JSON when we pushed a large of data. ObjectMapper can not read string json.
+    // So we have to use this method to parse string and build a json string as ReportResponse
+    // E.g invalid json response from MailChimp https://gist.github.com/thangnc/dc94026e4b13b728b7303f402b458b05
+    private JsonNode parseInvalidJsonString(final String json)
+    {
+        int totalCreatedIndex = json.indexOf("\"total_created\"");
+        int totalUpdatedIndex = json.indexOf("\"total_updated\"");
+        int errorCountIndex = json.indexOf("\"error_count\"");
+        int errorsIndex = json.indexOf("\"errors\"");
+
+        StringBuilder validJson = new StringBuilder();
+        validJson.append("{").append(json.substring(errorsIndex, totalCreatedIndex - 1)).append(",");
+        validJson.append(json.substring(totalCreatedIndex, totalCreatedIndex + "\"total_created\"".length() + 2)).append(",");
+        validJson.append(json.substring(totalUpdatedIndex, totalUpdatedIndex + "\"total_updated\"".length() + 2)).append(",");
+        validJson.append(json.substring(errorCountIndex, errorCountIndex + "\"error_count\"".length() + 2)).append("}");
+
+        try {
+            return this.jsonMapper.readTree(validJson.toString());
+        }
         catch (IOException ex) {
             throw new DataException(ex);
         }
     }
 
+    public void avoidFlushAPI(String reason)
+    {
+        try {
+            LOG.info("{} in 5s...", reason);
+            Thread.sleep(5000);
+        }
+        catch (InterruptedException e) {
+            LOG.warn("Failed to sleep: {}", e.getMessage());
+        }
+    }
+
     /**
      * MailChimp API v3 supports non expires access_token. Then no need refresh_token
      *

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.3.19
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-19 00:00:00.000000000 Z
+date: 2017-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Dumps records to MailChimp.
 email:
 - thang@treasure-data.com
@@ -88,13 +88,13 @@ files:
 - test/embulk/output/test_mailchimp.rb
 - test/override_assert_raise.rb
 - test/run-test.rb
-- classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-output-mailchimp-0.3.18.jar
-- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
-- classpath/jetty-client-9.2.14.v20151106.jar
-- classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/embulk-output-mailchimp-0.3.19.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
+- classpath/jetty-http-9.2.14.v20151106.jar
+- classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 homepage: https://github.com/treasure-data/embulk-output-mailchimp
 licenses:
 - MIT