embulk-output-mailchimp 0.3.22 → 0.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fc26e821ca35cc96b7cb4b5525f53dd73643e2c
-  data.tar.gz: b895e9c6c3936faa4f07689cb270b3d312326d08
+  metadata.gz: f18ee80430139a3e22677d7e18e3fd80eed89665
+  data.tar.gz: 1c3ebcd7abeaae441d926103f0920ca7d5d6ac6d
 SHA512:
-  metadata.gz: f0e97a31cd7b726fc993c9b96c79ff244e5b885ac9de3d68e2503c6beb2def9d5e2ad045c6c472fd58cfe8c7b61172ef56a4c4acde6fa62bdb841e017648780f
-  data.tar.gz: eb162f943bebab1a3f83c584659d5ac092dc4c057b787c8e44e15fb370e40c9bdfee179770cf59f9d5da29a32833ab07145af7672cf38f67349450cf8008215a
+  metadata.gz: ae75be9b1e060a13d6f0bcb4dfa71cea026f653832944bbcfd023df5987b819407decfac00fbdb84c8f31431c7c30dfe1dd8f2b16dcfa47f813ada7a93ba8a64
+  data.tar.gz: 960ede6f290226c7f35d3d2554175669e14db0a2ecc8ec89f87a8e2548b63f4438b54651f1e04a5a4122853709473a8a15acbae35e5f9e654cda87a62b062200

data/.travis.yml

@@ -1,7 +1,7 @@
 dist: precise
 language: java
 jdk:
-  - openjdk7
+  - oraclejdk8
 script:
   - ./gradlew clean checkstyle check jacocoTestReport
 addons:

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.23 - 2018-09-12
+- Introduce an option to fail the job when there is an error returning from Mailchimp. Previous versions marked the job as success with
+detail error in log
 ## 0.3.22 - 2018-03-07
 - Fixed bug NPE when checking interest categories [#41](https://github.com/treasure-data/embulk-output-mailchimp/pull/41)
 

data/README.md

@@ -25,6 +25,7 @@ add e-mail to List in MailChimp.
 - **grouping_columns**: Array for group names in MailChimp dashboard(array, default: nil)
 - **language_column**: column name for language (string, optional, default: nil)
 - **double_optin**: control whether to send an opt-in confirmation email (boolean, default: true)
+- **atomic_upsert** : Control the atomicity for the job. Job will be marked as success only when there is no error from Mailchimp. Default as false.
 - **max_records_per_request**: The max records per batch request. MailChimp API enables max records is 500 per batch request (int, default: 500)
 - **sleep_between_requests_millis**: The time to sleep between requests to avoid flood MailChimp API (int, default: 3000)
 
@@ -38,6 +39,7 @@ out:
   list_id: 'XXXXXXXXXX'
   update_existing: false
   double_optin: false
+  atomic_upsert: false
   email_column: e-mail
   fname_column: first name
   lname_column: lname

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.3.22"
+version = "0.3.23"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java

@@ -1,6 +1,7 @@
 package org.embulk.output.mailchimp;
 
 import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
 import org.embulk.base.restclient.RestClientOutputPluginDelegate;
 import org.embulk.base.restclient.RestClientOutputTaskBase;
 import org.embulk.base.restclient.jackson.JacksonServiceRequestMapper;
@@ -13,6 +14,7 @@ import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskReport;
 import org.embulk.output.mailchimp.model.AuthMethod;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
@@ -96,6 +98,10 @@ public class MailChimpOutputPluginDelegate
         @ConfigDefault("false")
         boolean getUpdateExisting();
 
+        @Config("atomic_upsert")
+        @ConfigDefault("false")
+        boolean getAtomicUpsert();
+
         @Config("replace_interests")
         @ConfigDefault("true")
         boolean getReplaceInterests();
@@ -146,6 +152,9 @@ public class MailChimpOutputPluginDelegate
         if (!checkExistColumns(schema, task.getEmailColumn(), task.getFnameColumn(), task.getLnameColumn())) {
             throw new ConfigException("Columns ['email', 'fname', 'lname'] must not be null or empty string");
         }
+        if (task.getAtomicUpsert()) {
+            LOG.info(" Treating upsert as atomic operation");
+        }
     }
 
     @Override
@@ -167,14 +176,21 @@ public class MailChimpOutputPluginDelegate
                                       final List<TaskReport> taskReports)
     {
         long totalInserted = 0;
+        int totalError = 0;
         for (TaskReport taskReport : taskReports) {
             if (taskReport.has("pushed")) {
                 totalInserted += taskReport.get(Long.class, "pushed");
             }
+            if (taskReport.has("error_count")) {
+                totalError += taskReport.get(Integer.class, "error_count");
+            }
         }
-
         LOG.info("Pushed completed. {} records", totalInserted);
-
+        // When atomic upsert is true, client expects all records are done properly.
+        if (task.getAtomicUpsert() && totalError > 0) {
+            LOG.info("Job requires atomic operation for all records. And there were {} errors in processing => Error as job's status", totalError);
+            throw Throwables.propagate(new DataException("Some records are not properly processed at MailChimp. See log for detail"));
+        }
         return Exec.newConfigDiff();
     }
 }

data/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java

@@ -54,6 +54,7 @@ public class MailChimpRecordBuffer
     private final ObjectMapper mapper;
     private final Schema schema;
     private int requestCount;
+    private int errorCount;
     private long totalCount;
     private List<JsonNode> records;
     private Map<String, Map<String, InterestResponse>> categories;
@@ -129,8 +130,7 @@ public class MailChimpRecordBuffer
                 uniqueRecords = new ArrayList<>();
                 duplicatedRecords = new ArrayList<>();
             }
-
-            return Exec.newTaskReport().set("pushed", totalCount);
+            return Exec.newTaskReport().set("pushed", totalCount).set("error_count", errorCount);
         }
         catch (JsonProcessingException jpe) {
             throw new DataException(jpe);
@@ -304,6 +304,7 @@ public class MailChimpRecordBuffer
                  records.size(), reportResponse.getTotalCreated(),
                  reportResponse.getTotalUpdated(),
                  reportResponse.getErrorCount(), System.currentTimeMillis() - startTime);
+        errorCount += reportResponse.getErrors().size();
         mailChimpClient.handleErrors(reportResponse.getErrors());
 
         mailChimpClient.avoidFloodAPI("Process next request", task.getSleepBetweenRequestsMillis());

data/src/test/java/org/embulk/output/mailchimp/TestMailChimpOutputPlugin.java

@@ -98,6 +98,12 @@ public class TestMailChimpOutputPlugin
         ConfigSource config = baseConfig.set("list_id", "");
         doSetUpSchemaAndRun(config, plugin);
     }
+    @Test(expected = ConfigException.class)
+    public void test_config_atomicUpsert()
+    {
+        ConfigSource config = baseConfig.set("atomic_upsert", true);
+        doSetUpSchemaAndRun(config, plugin);
+    }
 
     @Test(expected = ConfigException.class)
     public void test_config_invalidWithColumnEmailRequires()

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.3.22
+  version: 0.3.23
 platform: ruby
 authors:
 - Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-07 00:00:00.000000000 Z
+date: 2018-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Dumps records to MailChimp.
 email:
 - thang@treasure-data.com
@@ -88,13 +88,13 @@ files:
 - test/embulk/output/test_mailchimp.rb
 - test/override_assert_raise.rb
 - test/run-test.rb
-- classpath/jetty-io-9.2.14.v20151106.jar
-- classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-output-mailchimp-0.3.22.jar
-- classpath/jetty-http-9.2.14.v20151106.jar
-- classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-output-mailchimp-0.3.23.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
+- classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/jetty-http-9.2.14.v20151106.jar
+- classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/jetty-util-9.2.14.v20151106.jar
 homepage: https://github.com/treasure-data/embulk-output-mailchimp
 licenses:
 - MIT