embulk-output-mailchimp 0.2.3 → 0.3.2

data/src/main/java/org/embulk/output/mailchimp/validation/ColumnDataValidator.java

@@ -0,0 +1,40 @@
+package org.embulk.output.mailchimp.validation;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+
+import java.util.Arrays;
+
+/**
+ * An Util class to validate data based on @{@link Schema} to build data payload
+ * <p>
+ * Created by thangnc on 4/18/17.
+ */
+public final class ColumnDataValidator
+{
+    private ColumnDataValidator()
+    {
+    }
+
+    /**
+     * Check required columns. Should require `email` and `status` columns
+     *
+     * @param schema       the schema
+     * @param allowColumns the columns
+     * @return the boolean
+     */
+    public static boolean checkExistColumns(final Schema schema, final String... allowColumns)
+    {
+        int found = 0;
+
+        for (Column column : schema.getColumns()) {
+            if (!Arrays.asList(allowColumns).contains(column.getName())) {
+                continue;
+            }
+
+            found++;
+        }
+
+        return found == allowColumns.length;
+    }
+}

data/src/test/java/org/embulk/output/mailchimp/CircleCICredentials.java

@@ -0,0 +1,22 @@
+package org.embulk.output.mailchimp;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigSource;
+
+/**
+ * Created by thangnc on 3/6/17.
+ */
+public class CircleCICredentials
+{
+    private CircleCICredentials()
+    {
+    }
+
+    public static ConfigSource credentials()
+    {
+        return EmbulkEmbed.newSystemConfigLoader().newConfigSource()
+                .set("apikey", System.getenv("MAILCHIMP_APIKEY"))
+                .set("access_token", System.getenv("MAILCHIMP_ACCESS_TOKEN"))
+                .set("list_id", System.getenv("MAILCHIMP_LIST_ID"));
+    }
+}

data/src/test/java/org/embulk/output/mailchimp/TestColumnDataValidator.java

@@ -0,0 +1,43 @@
+package org.embulk.output.mailchimp;
+
+import org.embulk.output.mailchimp.validation.ColumnDataValidator;
+import org.embulk.spi.Schema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.embulk.spi.type.Types.STRING;
+
+/**
+ * Created by thangnc on 5/12/17.
+ */
+public class TestColumnDataValidator
+{
+    @Test
+    public void test_checkExistColumns_valid()
+    {
+        Schema schema = Schema.builder()
+                .add("email", STRING)
+                .add("fname", STRING)
+                .add("lname", STRING)
+                .add("status", STRING)
+                .build();
+
+        Assert.assertEquals("Column should be exists",
+                            true,
+                            ColumnDataValidator.checkExistColumns(schema, "status", "email"));
+    }
+
+    @Test
+    public void test_checkExistColumns_invalid()
+    {
+        Schema schema = Schema.builder()
+                .add("email", STRING)
+                .add("fname", STRING)
+                .add("lname", STRING)
+                .build();
+
+        Assert.assertEquals("Column should be not exist",
+                            false,
+                            ColumnDataValidator.checkExistColumns(schema, "status", "email"));
+    }
+}

data/src/test/java/org/embulk/output/mailchimp/TestMailChimpHelper.java

@@ -0,0 +1,54 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.google.common.collect.Multimap;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.containsCaseInsensitive;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.extractMemberStatus;
+import static org.embulk.output.mailchimp.helper.MailChimpHelper.maskEmail;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by thangnc on 4/26/17.
+ */
+public class TestMailChimpHelper
+{
+    @Test
+    public void test_maskEmail_validInErrorResponse()
+    {
+        String given = "thang0001@example.com looks fake or invalid, please enter a real email address.";
+        String expect = "t******01@example.com looks fake or invalid, please enter a real email address.";
+        assertEquals("Email should match", expect, maskEmail(given));
+    }
+
+    @Test
+    public void test_containsCaseInsensitive_validMergeFields()
+    {
+        String expect = "United State";
+        assertEquals("Interest category should match", expect, containsCaseInsensitive("united state",
+                                                                                       Arrays.asList("Donating", "United State")));
+    }
+
+    @Test
+    public void test_extractMemberStatus()
+    {
+        List<JsonNode> given = new ArrayList<>();
+        given.add(JsonNodeFactory.instance.objectNode().put("status", "subcribed").put("email", "thang@gmail.com"));
+        given.add(JsonNodeFactory.instance.objectNode().put("status", "pending").put("email", "thang123@gmail.com"));
+        given.add(JsonNodeFactory.instance.objectNode().put("status", "abc").put("email", "thang456@gmail.com"));
+        given.add(JsonNodeFactory.instance.objectNode().put("status", "pending").put("email", "thang789@gmail.com"));
+        given.add(JsonNodeFactory.instance.objectNode().put("status", "subcribed").put("email", "thang000@gmail.com"));
+
+        Multimap<String, JsonNode> statusMap = extractMemberStatus(given);
+        assertEquals("Status should match", 3, statusMap.keySet().size());
+        assertEquals("Status should contain keys", true, statusMap.containsKey("pending"));
+        assertEquals("Status should contain keys", true, statusMap.containsKey("subcribed"));
+        assertEquals("Status should contain keys", true, statusMap.containsKey("abc"));
+    }
+}

data/src/test/java/org/embulk/output/mailchimp/TestMailChimpOutputPlugin.java

@@ -0,0 +1,158 @@
+package org.embulk.output.mailchimp;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.guava.GuavaModule;
+import com.fasterxml.jackson.datatype.joda.JodaModule;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ModelManager;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.embulk.spi.type.Types.STRING;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assume.assumeThat;
+
+/**
+ * Created by thangnc on 4/14/17.
+ */
+public class TestMailChimpOutputPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource baseConfig;
+    private MailChimpOutputPluginDelegate.PluginTask task;
+    private MailChimpOutputPlugin plugin;
+
+    @Before
+    public void setup()
+    {
+        plugin = new MailChimpOutputPlugin();
+        baseConfig = config();
+        task = baseConfig.loadConfig(MailChimpOutputPluginDelegate.PluginTask.class);
+    }
+
+    @After
+    public void teardown()
+    {
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithEmptyApiKey()
+    {
+        ConfigSource config = baseConfig.set("apikey", "");
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithNullApiKey()
+    {
+        ConfigSource config = baseConfig.set("apikey", null);
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithEmptyAccessToken()
+    {
+        ConfigSource config = baseConfig.set("auth_method", "oauth")
+                .set("access_token", "");
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithNullAccessToken()
+    {
+        ConfigSource config = baseConfig.set("auth_method", "oauth")
+                .set("access_token", null);
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithNullListId()
+    {
+        ConfigSource config = baseConfig.set("list_id", null);
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithEmptyListId()
+    {
+        ConfigSource config = baseConfig.set("list_id", "");
+        doSetUpSchemaAndRun(config, plugin);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void test_config_invalidWithColumnEmailRequires()
+    {
+        ConfigSource config = baseConfig;
+        Schema schema = Schema.builder()
+                .add("fname", STRING)
+                .add("lname", STRING)
+                .build();
+
+        final TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
+        output.finish();
+
+        plugin.transaction(config, schema, 0, new OutputControl());
+    }
+
+    /**
+     * Load plugin config with Guava & Joda support
+     */
+    private static ConfigSource config()
+    {
+        String path = System.getenv("EMBULK_OUTPUT_MAILCHIMP_TEST_CONFIG");
+        assumeThat(isNullOrEmpty(path), is(false));
+        try {
+            ObjectMapper mapper = new ObjectMapper()
+                    .registerModule(new GuavaModule())
+                    .registerModule(new JodaModule());
+            ConfigLoader configLoader = new ConfigLoader(new ModelManager(null, mapper));
+            return configLoader.fromYamlFile(new File(path));
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    private class OutputControl implements OutputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource)
+        {
+            return Lists.newArrayList(Exec.newTaskReport());
+        }
+    }
+
+    private void doSetUpSchemaAndRun(final ConfigSource config, final MailChimpOutputPlugin plugin)
+    {
+        Schema schema = Schema.builder()
+                .add("email", STRING)
+                .add("fname", STRING)
+                .add("lname", STRING)
+                .build();
+
+        final TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
+        output.finish();
+
+        plugin.transaction(config, schema, 0, new OutputControl());
+    }
+}

data/src/test/resources/csv/email.csv

@@ -0,0 +1,2 @@
+email;fistname;lastname;favorite;type
+hi@example.com;first_name;last_name;music,book;middle

metadata

@@ -1,76 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mailchimp
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.2
 platform: ruby
 authors:
-- uu59
-- takkanm
+- Thang Nguyen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2017-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.6
-  name: mailchimp-api
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.0.6
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  name: perfect_retry
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.9
-  name: embulk
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.9
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.10.6
+        version: '1.0'
   name: bundler
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.10.6
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -78,88 +35,64 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: test-unit
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: test-unit-rr
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  name: simplecov
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: coveralls
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Dumps records to Mailchimp.
+description: Dumps records to MailChimp.
 email:
-- k@uu59.org
-- takkanm@gmail.com
+- thang@treasure-data.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coveralls.yml"
-- ".gitignore"
-- ".ruby-version"
-- ".travis.yml"
+- .coveralls.yml
+- .gitignore
+- .travis.yml
 - CHANGELOG.md
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
-- embulk-output-mailchimp.gemspec
+- build.gradle
+- ci/travis_mailchimp.yml
+- circle.yml
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
 - lib/embulk/output/mailchimp.rb
+- src/main/java/org/embulk/output/mailchimp/MailChimpAbstractRecordBuffer.java
+- src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java
+- src/main/java/org/embulk/output/mailchimp/MailChimpOutputPlugin.java
+- src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java
+- src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java
+- src/main/java/org/embulk/output/mailchimp/helper/MailChimpHelper.java
+- src/main/java/org/embulk/output/mailchimp/model/AuthMethod.java
+- src/main/java/org/embulk/output/mailchimp/model/CategoriesResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/ErrorResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/InterestResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/InterestsResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/MemberStatus.java
+- src/main/java/org/embulk/output/mailchimp/model/MetaDataResponse.java
+- src/main/java/org/embulk/output/mailchimp/model/ReportResponse.java
+- src/main/java/org/embulk/output/mailchimp/validation/ColumnDataValidator.java
+- src/test/java/org/embulk/output/mailchimp/CircleCICredentials.java
+- src/test/java/org/embulk/output/mailchimp/TestColumnDataValidator.java
+- src/test/java/org/embulk/output/mailchimp/TestMailChimpHelper.java
+- src/test/java/org/embulk/output/mailchimp/TestMailChimpOutputPlugin.java
+- src/test/resources/csv/email.csv
 - test/embulk/output/test_mailchimp.rb
 - test/override_assert_raise.rb
 - test/run-test.rb
-homepage:
+- classpath/embulk-base-restclient-0.5.0.jar
+- classpath/embulk-output-mailchimp-0.3.2.jar
+- classpath/embulk-util-retryhelper-jetty92-0.5.0.jar
+- classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/jetty-http-9.2.14.v20151106.jar
+- classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/jetty-util-9.2.14.v20151106.jar
+homepage: https://github.com/treasure-data/embulk-output-mailchimp
 licenses:
 - MIT
 metadata: {}
@@ -169,17 +102,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Mailchimp output plugin for Embulk

data/.ruby-version

@@ -1 +0,0 @@
-jruby-9.0.0.0

data/Gemfile

@@ -1,2 +0,0 @@
-source 'https://rubygems.org/'
-gemspec

data/Rakefile

@@ -1,29 +0,0 @@
-require "bundler/gem_tasks"
-
-task default: :build
-
-task 'clean' do
-  Dir.glob("pkg/*.gem").each do |name|
-    sh "rm #{name}"
-  end
-end
-
-task 'reload' do
-  spec = Bundler::GemHelper.gemspec
-
-  sh "embulk gem uninstall #{spec.name}"
-  Rake::Task['clean'].execute
-  Rake::Task['build'].execute
-  sh "embulk gem install pkg/#{spec.name}-#{spec.version}.gem"
-end
-
-desc 'Run tests'
-task :test do
-  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
-end
-
-desc "Run tests with coverage"
-task :cov do
-  ENV["COVERAGE"] = "1"
-  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
-end

data/embulk-output-mailchimp.gemspec

@@ -1,27 +0,0 @@
-
-Gem::Specification.new do |spec|
-  spec.name          = "embulk-output-mailchimp"
-  spec.version       = "0.2.3"
-  spec.authors       = ["uu59", "takkanm"]
-  spec.summary       = "Mailchimp output plugin for Embulk"
-  spec.description   = "Dumps records to Mailchimp."
-  spec.email         = ["k@uu59.org", "takkanm@gmail.com"]
-  spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/takkanm/embulk-output-mailchimp"
-
-  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
-  spec.require_paths = ["lib"]
-
-  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
-  spec.add_dependency 'mailchimp-api', ['~> 2.0.6']
-  spec.add_dependency 'perfect_retry', "~> 0.3"
-
-  spec.add_development_dependency 'embulk', ['~> 0.7.9']
-  spec.add_development_dependency 'bundler', ['~> 1.10.6']
-  spec.add_development_dependency 'rake', ['>= 10.0']
-  spec.add_development_dependency 'test-unit'
-  spec.add_development_dependency 'test-unit-rr'
-  spec.add_development_dependency 'simplecov', ['>= 0.9']
-  spec.add_development_dependency 'coveralls'
-end