embulk-output-mailchimp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 45060c5aec8200c9d1806d466414c014834dec45
+  data.tar.gz: 975cf885d3d90f9de70c7b9a119a96a4753ecfe9
+SHA512:
+  metadata.gz: ca60d58b8698b777e14df5ced2e3ff82543d4a42979c8a9ac2fff606852e6249131956d0b3c6f8f7b3c51d37c0b7a22da68de6cfd2be746f24c3b0cd04c0eb4b
+  data.tar.gz: 940b9ce3f294f45e3d5797b64df87a6003c8ff413c3441122a66f447dfd64869d5d066daef91e5a047425b1ee171de7577c1683d1acecae08699f9cae5789429

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock
+/coverage/

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.0.0

data/.travis.yml

@@ -0,0 +1,9 @@
+language: jruby
+before_install:
+    - rvm get head --auto-dotfiles
+    - rvm install jruby-9.0.1.0
+    - gem install bundler
+    - gem update bundler
+    - bundle install
+script:
+  - bundle exec rake cov

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# Mailchimp output plugin for Embulk
+[![Coverage Status](https://coveralls.io/repos/treasure-data/embulk-output-mailchimp/badge.svg?branch=master&service=github)](https://coveralls.io/github/treasure-data/embulk-output-mailchimp?branch=master)
+[![Build Status](https://travis-ci.org/treasure-data/embulk-output-mailchimp.svg)](https://travis-ci.org/treasure-data/embulk-output-mailchimp?branch=master)
+
+add e-mail to List in MailChimp.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: no
+
+## Configuration
+
+- **apikey**: Mailchimp API key (string, required)
+- **list_id**: Mailchimp List id (string, required)
+- **double_optin**: control whether to send an opt-in confirmation email (boolean, default: true)
+- **update_existing**: control whether to update members that are already subscribed to the list or to return an error (boolean, default: false)
+- **replace_interests**: determine whether we replace the interest groups with the updated groups provided, or we add the provided groups to the member's interest groups (boolean, default: true)
+- **email_column**: column name for email (string, default: 'email')
+- **fname_column**: column name for first name (string, default: 'fname')
+- **lname_column**: column name for last name(string, default: 'lname')
+
+## Example
+
+```yaml
+out:
+  type: mailchimp
+  apikey: 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX-XXX'
+  list_id: 'XXXXXXXXXX'
+  double_optin: false
+  update_existing: false
+  replace_interests: true
+  email_column: 'e-mail'
+  fname_column: 'first name'
+  lname_column: 'lname'
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,29 @@
+require "bundler/gem_tasks"
+
+task default: :build
+
+task 'clean' do
+  Dir.glob("pkg/*.gem").each do |name|
+    sh "rm #{name}"
+  end
+end
+
+task 'reload' do
+  spec = Bundler::GemHelper.gemspec
+
+  sh "embulk gem uninstall #{spec.name}"
+  Rake::Task['clean'].execute
+  Rake::Task['build'].execute
+  sh "embulk gem install pkg/#{spec.name}-#{spec.version}.gem"
+end
+
+desc 'Run tests'
+task :test do
+  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+desc "Run tests with coverage"
+task :cov do
+  ENV["COVERAGE"] = "1"
+  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
+end

data/embulk-output-mailchimp.gemspec

@@ -0,0 +1,27 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-mailchimp"
+  spec.version       = "0.1.0"
+  spec.authors       = ["takkanm"]
+  spec.summary       = "Mailchimp output plugin for Embulk"
+  spec.description   = "Dumps records to Mailchimp."
+  spec.email         = ["takkanm@gmail.com"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/takkanm/embulk-output-mailchimp"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_dependency 'mailchimp-api', ['~> 2.0.6']
+  spec.add_dependency 'perfect_retry', "~> 0.3"
+
+  spec.add_development_dependency 'embulk', ['~> 0.7.9']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'simplecov', ['>= 0.9']
+  spec.add_development_dependency 'coveralls'
+end

data/lib/embulk/output/mailchimp.rb

@@ -0,0 +1,142 @@
+require 'mailchimp'
+require 'perfect_retry'
+
+module Embulk
+  module Output
+
+    class Mailchimp < OutputPlugin
+      Plugin.register_output("mailchimp", self)
+
+      class Client
+        def initialize(apikey)
+          @client = ::Mailchimp::API.new(apikey)
+        rescue ::Mailchimp::InvalidApiKeyError => e
+          raise Embulk::ConfigError.new(e.message)
+        end
+
+        def batch_subscribe_list(list_id, subscribers, double_optin, update_existing, replace_interests)
+          @client.lists.batch_subscribe(list_id, subscribers, double_optin, update_existing, replace_interests)
+        rescue ::Mailchimp::UserUnderMaintenanceError, ::Mailchimp::TooManyConnectionsError => e
+          Embulk.logger.warn e.message
+          raise e
+        rescue ::Mailchimp::Error => e
+          raise Embulk::DataError.new(e.message)
+        end
+      end
+
+      MAX_EMAIL_COUNT = 1_000_000
+
+      def self.transaction(config, schema, count, &control)
+        task = {
+          apikey:                 config.param("apikey",                 :string),
+          list_id:                config.param("list_id",                :string),
+          double_optin:           config.param("double_optin",           :bool,    default: true),
+          update_existing:        config.param("update_existing",        :bool,    default: false),
+          replace_interests:      config.param("replace_interests",      :bool,    default: true),
+          email_column:           config.param("email_column",           :string,  default: "email"),
+          fname_column:           config.param("fname_column",           :string,  default: "fname"),
+          lname_column:           config.param("lname_column",           :string,  default: "lname"),
+          retry_limit:            config.param("retry_limit",            :integer, default: 5),
+          retry_initial_wait_sec: config.param("retry_initial_wait_sec", :integer, default: 1),
+          stop_on_invalid_record: config.param("stop_on_invalid_record", :bool,    default: true),
+        }
+
+        Client.new(task[:apikey]) # NOTE for validate apikey
+        raise Embulk::ConfigError.new("schema has no '#{task[:email_column]}' column") if schema.none? {|s| s.name == task[:email_column] }
+
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def init
+        @client            = Client.new(task[:apikey])
+        @list_id           = task[:list_id]
+        @double_optin      = task[:double_optin]
+        @update_existing   = task[:update_existing]
+        @replace_interests = task[:replace_interests]
+        @email_column      = task[:email_column]
+        @fname_column      = task[:fname_column]
+        @lname_column      = task[:lname_column]
+        @subscribers       = []
+
+        @retry_manager = PerfectRetry.new do |config|
+          config.limit = task[:retry_limit]
+          config.sleep = lambda{|n| task[:retry_initial_wait_sec] * (2 ** (n - 1)) }
+          config.logger = Embulk.logger
+          config.log_level = nil
+          config.dont_rescues = [Embulk::ConfigError, Embulk::DataError]
+        end
+        @stop_on_invalid_record = task[:stop_on_invalid_record]
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        page.each do |record|
+          row = Hash[schema.names.zip(record)]
+          if row[@email_column]
+            add_subscriber row
+
+            flush_subscribers! unless @subscribers.size < MAX_EMAIL_COUNT
+          else
+            raise Embulk::DataError.new("#{@email_column} is empty") if @stop_on_invalid_record
+          end
+        end
+      end
+
+      def finish
+        flush_subscribers!
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+
+      private
+
+      def add_subscriber(row)
+        return unless row[@email_column]
+
+        merge_columns = {fname: @fname_column, lname: @lname_column}
+
+        merge_vars = merge_columns.each_with_object({}) do |(key, col_name), m|
+          m[key] = row[col_name] if row[col_name]
+        end
+
+        @subscribers << {
+          EMAIL: {
+            email: row[@email_column]
+          },
+          merge_vars: merge_vars,
+        }
+      end
+
+      def flush_subscribers!
+        return if @subscribers.empty?
+
+        send_subscribers!
+        @subscribers = []
+      end
+
+      def send_subscribers!
+        @retry_manager.with_retry do
+          @client.batch_subscribe_list(
+            @list_id,
+            @subscribers,
+            @double_optin,
+            @update_existing,
+            @replace_interests
+          )
+        end
+      end
+    end
+
+  end
+end

data/test/embulk/output/test_mailchimp.rb

@@ -0,0 +1,165 @@
+require "embulk"
+Embulk.setup
+
+require "embulk/output/mailchimp"
+require "override_assert_raise"
+
+
+module Embulk
+  module Output
+    class TestMailchimp < Test::Unit::TestCase
+      include OverrideAssertRaise
+
+      def apikey
+        'XXXXXXXXXXXXXXXXXXXXXX'
+      end
+
+      def list_id
+        'listlist'
+      end
+
+      class TestTransaction < self
+        def base_config
+          {
+            type: 'mailchimp',
+            apikey: apikey,
+            list_id: list_id,
+          }
+        end
+
+        class MailchimpAPIKeyIsValid < self
+          def setup
+            stub(Embulk::Output::Mailchimp::Client).new(apikey) { }
+          end
+
+          def test_call_finish_without_error
+            source = create_config(base_config)
+            schema = create_schema([{name: 'email', type: :string}])
+
+            Embulk::Output::Mailchimp.transaction(source, schema, 1) {}
+          end
+
+          def test_schema_has_no_email_column
+            source = create_config(base_config.merge(email_column: 'e_mail'))
+            schema = create_schema([{name: 'email', type: :string}])
+
+            assert_raise(Embulk::ConfigError) do
+              Embulk::Output::Mailchimp.transaction(source, schema, 1) {}
+            end
+          end
+        end
+
+        class MailchimpAPIKeyIsInvalid < self
+          def setup
+            stub(::Mailchimp::API).new(apikey) { raise ::Mailchimp::InvalidApiKeyError }
+          end
+
+          def test_call_finish_without_error
+            source = create_config(base_config)
+            schema = create_schema([{name: 'email', type: :string}])
+
+            assert_raise(Embulk::ConfigError) do
+              Embulk::Output::Mailchimp.transaction(source, schema, 1) {}
+            end
+          end
+        end
+      end
+
+      class TestAdd < self
+        setup do
+          stub(::Mailchimp::API).new(apikey) { }
+        end
+
+        def create_mailchimp(optional_task = {})
+          Mailchimp.new(
+            {apikey: apikey, list_id: list_id, email_column: 'email'}.merge(optional_task),
+            create_schema([
+              {name: 'email', type: :string},
+              {name: 'fname', type: :string},
+              {name: 'lname', type: :string},
+            ]),
+            1
+          )
+        end
+
+        class MergeVars < self
+          def page
+            [['a@example.com', 'first', 'last']]
+          end
+
+          def test_stop_on_invalid_record_is_false
+            mailchimp = create_mailchimp(
+              lname_column: 'lname',
+              fname_column: 'fname'
+            )
+
+            mailchimp.add(page)
+            subscriber = mailchimp.instance_variable_get(:@subscribers)[0]
+
+            assert_equal(subscriber[:EMAIL][:email], 'a@example.com')
+            assert_equal(subscriber[:merge_vars][:fname], 'first')
+            assert_equal(subscriber[:merge_vars][:lname], 'last')
+          end
+        end
+
+        # NOTE skip until const mocking
+        class RecordeOverMaxEmailCount# < self
+          def page
+            [['@example.com']] * Mailchimp::MAX_EMAIL_COUNT
+          end
+
+          def test_flush_if_over_max_email_count
+            mailchimp = create_mailchimp
+            mock(mailchimp).send_subscribers!
+
+            mailchimp.add(page)
+            assert_equal(mailchimp.instance_variable_get(:@subscribers).size, 1)
+          end
+        end
+
+        class EmailColumnValueIsEmpty < self
+          def page
+            [
+              ['a@example.com'], [], ['c@example.com']
+            ]
+          end
+
+          def test_stop_on_invalid_record_is_false
+            mailchimp = create_mailchimp(stop_on_invalid_record: false)
+
+            mailchimp.add(page)
+            assert_equal(mailchimp.instance_variable_get(:@subscribers).size, 2)
+          end
+
+          def test_stop_on_invalid_record_is_true
+            mailchimp = create_mailchimp(stop_on_invalid_record: true)
+
+            assert_raise(Embulk::DataError) do
+              mailchimp.add(page)
+            end
+          end
+        end
+      end
+
+      private
+
+      def create_config(config)
+        Embulk::DataSource[*config.to_a.flatten(1)]
+      end
+
+      def create_schema(columns)
+        Embulk::Schema.new(columns.map {|c| Embulk::Column.new(c) })
+      end
+
+      def mute_logger
+        stub(Embulk).logger { Logger.new(File::NULL) }
+      end
+
+      def setup_plugin(specific_task = nil, specific_schema = nil)
+        @plugin = Mailchimp.new(specific_task || task, specific_schema || schema, 1)
+        stub(@plugin).task { specific_task || task }
+        @plugin.init
+      end
+    end
+  end
+end

data/test/override_assert_raise.rb

@@ -0,0 +1,18 @@
+module OverrideAssertRaise
+  # NOTE: Embulk 0.7.1+ required to raise ConfigError to do as `ConfigError.new("message")`,
+  #       original `assert_raise` method can't catch that, but `begin .. rescue` can.
+  #       So we override assert_raise as below.
+  def assert_raise(expected_class = StandardError, &block)
+    begin
+      block.call
+      assert_equal expected_class, nil
+    rescue ::Test::Unit::AssertionFailedError => e
+      # failed assert raises this Error and that extends StandardError, so rescue it first
+      raise e
+    rescue expected_class
+      assert true # passed
+    rescue => e
+      assert_equal(expected_class, e.class) # not expected one raised
+    end
+  end
+end

data/test/run-test.rb

@@ -0,0 +1,32 @@
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir  = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/rr"
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+if ENV['CI']
+  require 'simplecov'
+  require 'coveralls'
+  Coveralls.wear!
+
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[Coveralls::SimpleCov::Formatter]
+  SimpleCov.start 'test_frameworks'
+end
+
+if ENV["COVERAGE"]
+  require 'simplecov'
+  SimpleCov.start 'test_frameworks'
+
+  if ENV['CIRCLE_ARTIFACTS'] # https://circleci.com/docs/code-coverage
+    dir = File.join(ENV['CIRCLE_ARTIFACTS'], "coverage")
+    SimpleCov.coverage_dir(dir)
+  end
+end
+
+exit Test::Unit::AutoRunner.run(true, test_dir)

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-mailchimp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- takkanm
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-11-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.6
+  name: mailchimp-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.6
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: perfect_retry
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.9
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  name: simplecov
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: coveralls
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dumps records to Mailchimp.
+email:
+- takkanm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-mailchimp.gemspec
+- lib/embulk/output/mailchimp.rb
+- test/embulk/output/test_mailchimp.rb
+- test/override_assert_raise.rb
+- test/run-test.rb
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Mailchimp output plugin for Embulk
+test_files:
+- test/embulk/output/test_mailchimp.rb
+- test/override_assert_raise.rb
+- test/run-test.rb